Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: Manish 

'''

from bs4 import BeautifulSoup

from bson.binary import Binary

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder, HomeShopAffiliateInfo

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \

    Store as MStore, ungzipResponse, tprint

from dtr.storage import Mongo

from dtr.storage.Mongo import getImgSrc

from dtr.utils.utils import fetchResponseUsingProxy, PROXY_MESH_GENERAL

from pprint import pprint

from pymongo import MongoClient

import json

import pymongo

import re

import time

import traceback

import urllib

import urllib2

from urlparse import urlparse, parse_qs

import xml.etree.ElementTree as ET

from dtr.storage import MemCache

from dtr.storage.Mongo import getDealRank

AFFLIATE_TRASACTIONS_URL = "https://admin.optimisemedia.com/v2/reports/affiliate/leads/leadsummaryexport.aspx?Contact=796881&Country=26&Agency=95&Merchant=331902&Status=-1&Year=%d&Month=%d&Day=%d&EndYear=%d&EndMonth=%d&EndDay=%d&DateType=0&Sort=CompletionDate&Login=1347562DA5E3EFF6FB1561765C47C782&Format=XML&RestrictURL=0"

ORDER_TRACK_URL='https://m.homeshop18.com/order/orderDetail.mobi?orderId=%d'

HS_ORDER_TRACK_URL='http://www.homeshop18.com/track-your-order.html'

BASE_URL= 'http://www.shopclues.com'

BASE_MURL= 'http://m.shopclues.com'

BASE_PRODUCT_URL= 'http://m.homeshop18.com/product.mobi?productId=%d'

BASE_IMG_URL='http://stat.homeshop18.com/homeshop18'

#http://m.homeshop18.com/checkout/paySuccess.mobi?orderComplete=true

class Store(MStore):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      MStore.ORDER_PLACED : ['payment successful', 'new order - cod confirmation pending', 'processing', 'quality check','on schedule', 'processing - pickup initiated', 'processing - ready to dispatch','processing - procurement delay from merchant','processing - slight procurment delay from merchant','cod order confirmed by customer'],

                      MStore.ORDER_DELIVERED : ['delivered', 'complete'],

                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched','shipped','order handed to courier','order handed over to courier'],

                      MStore.ORDER_CANCELLED : ['payment failed', 'canceled', 'payment declined', 'order on hold - cancellation requested by customer', 'courier returned', 'canceled on customer request', 'canceled by customer','order canceled by customer','canceled - address not shippable','return complete','undelivered - returning to origin']

                      }

    OrderStatusConfirmationMap= {

                                 "P" : "Payment Successful",

                                 "D" : "Order Declined",

                                 "O" : "New Order - COD confirmation Pending"

                                 }

    OrderStatusStringMap = {

                            MStore.ORDER_PLACED : ['expect the order to reach', 'received your payment'],

                            MStore.ORDER_DELIVERED : ['has been delivered'],

                            MStore.ORDER_SHIPPED : ['has been shipped', 'has been dispatched'], 

                            MStore.ORDER_CANCELLED : ['has been cancelled', 'has been rejected','is returned back to us','payment failed']

                            }

    OrderStatusShownMap = {

                            "Under Process" : ['expect the order to reach', 'received your payment'],

                            "Order Delivered" : ['has been delivered'],

                            "Order Shipped" : ['has been shipped', 'has been dispatched'], 

                            "Order Cancelled" : ['has been cancelled', 'has been rejected','is returned back to us','payment failed']

                           }

    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def convertToObj(self,offer):

        orderRef = offer['MerchantRef']

        if len(orderRef)>15:

            orderRef = orderRef[0:len(orderRef)-10]

        offer1 = HomeShopAffiliateInfo(offer['UID'], offer['TransactionTime'], offer['TransactionID'], orderRef,  orderRef, offer['Merchant'], offer['PID'], offer['Product'], float(str(offer['SR'])), float(str(offer['TransactionValue'])), offer['UKey'], offer['ClickTime'], offer['Status'])

        return offer1

    def _saveToAffiliate(self, offers):

        collection = self.db.homeshopOrderAffiliateInfo

        mcollection = self.db.merchantOrder

        for offerObj in offers:

            offer = self.convertToObj(offerObj)

            collection.update({"transactionId":offer.transactionId, "subTagId":offer.subTagId, "payOut":offer.payOut},{"$set":todict(offer)}, upsert=True)

            mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True}, {"$set":{"subOrders.$.missingAff":False}})

    def scrapeAffiliate(self, startDate=datetime.today() - timedelta(days=10), endDate=datetime.today()):

        uri = AFFLIATE_TRASACTIONS_URL%(startDate.year,startDate.month,startDate.day,endDate.year,endDate.month,endDate.day)

        root = ET.parse(urllib2.urlopen(uri)).getroot()

        if len(root)> 0 and len(root[0])> 0:

            offers = []

            for child in root[0][0]:

                offers.append(child.attrib)

            self._saveToAffiliate(offers)

    def _setLastSaleDate(self, saleDate):

        self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})

    def getName(self):

        return "homeshop18"  

    def _getLastSaleDate(self,):

        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})

        if lastDaySaleObj is None:

            return datetime.min

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, statusList in Store.OrderStatusStringMap.iteritems():

            for value in statusList:

                if value in detailedStatus.lower():

                    return key

        print "Detailed Status need to be mapped", detailedStatus, self.store_id

        return None

    def _getDisplayStatusFromDetailedStatus(self, detailedStatus):

        for key, statusList in Store.OrderStatusShownMap.iteritems():

            for value in statusList:

                if value in detailedStatus.lower():

                    return key

        print "Display Status need to be mapped", detailedStatus, self.store_id

        return None

    def updateCashbackInSubOrders(self, subOrders):

        for subOrder in subOrders:

            cashbackStatus = Store.CB_NA

            cashbackAmount = 0

            percentage = 0

            amount = subOrder.amountPaid

            if amount > 0:

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)

                if cashbackAmount > 0:

                    cashbackStatus = Store.CB_PENDING

            subOrder.cashBackStatus = cashbackStatus

            subOrder.cashBackAmount = cashbackAmount

            subOrder.cashBackPercentage = percentage

        return subOrders

    def _parseUsingOrderJson(self, orderId, subTagId, userId, rawHtmlSoup, orderSuccessUrl):

        orderObj = None

        scripts = rawHtmlSoup.find_all('script')

        for script in scripts:

            if 'var order =' in script.text:

                requiredObjList = script.text.strip().split('\n')

                for val in requiredObjList:

                    if "$.parseJSON('" in val:

                        val = val.encode("ascii","ignore")

                        print 'Val......',val

                        print val.split("$.parseJSON('")[1].split("');")[0]

                        orderObj = json.loads(val.split("$.parseJSON('")[1].split("');")[0])

                        print orderObj

                        break

                break

        if orderObj is not None:

            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

            merchantOrder.placedOn = orderObj['orderDate']

            merchantOrder.merchantOrderId = str(long(orderObj['orderId']))

            merchantOrder.paidAmount = long(orderObj['pricing']['orderNetPrice'])

            merchantOrder.totalAmount = long(orderObj['pricing']['orderGrossPrice'])

            merchantOrder.discountApplied = long(orderObj['pricing']['discountCouponRedemptionAmount'])+long(orderObj['pricing']['giftCouponRedemptionAmount'])

            merchantOrder.deliveryCharges = long(orderObj['totalShipmentCharges'])

            subOrders= []

            for subOrderObj in orderObj['subOrders']:

                subOrder = SubOrder(subOrderObj['cartItem']['cartItemTitle'], BASE_PRODUCT_URL%(long(subOrderObj['cartItem']['productId'])), orderObj['orderDate'], long(subOrderObj['pricing']['payablePrice']))

                subOrder.estimatedDeliveryDate = subOrderObj['shipment']['expectedDeliveryDate']

                subOrder.merchantSubOrderId = str(subOrderObj['subOrderId'])

                if rawHtmlSoup.body.find("div", {'class':'sub-order-status'}) is not None:

                    subOrder.detailedStatus = rawHtmlSoup.body.find("div", {'class':'sub-order-status'}).text

                else:

                    subOrder.detailedStatus = 'Order Placed'

                subOrder.imgUrl = BASE_IMG_URL+subOrderObj['cartItem']['lineItemImageUrl']

                subOrder.offerDiscount = (long(subOrderObj['cartItem']['price'])+long(subOrderObj['shipment']['shipmentCharge']))*long(subOrderObj['cartItem']['itemQuantity'])-long(subOrderObj['pricing']['payablePrice'])

                subOrder.unitPrice = long(subOrderObj['cartItem']['price'])

                subOrder.productCode = str(long(subOrderObj['cartItem']['productId']))

                subOrder.amountPaid = long(subOrderObj['pricing']['payablePrice'])

                subOrder.quantity = long(subOrderObj['cartItem']['itemQuantity'])

                subOrder.tracingkUrl = ORDER_TRACK_URL%(long(orderObj['orderId']))

                dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

                subOrder.dealRank = dealRank.get('rank')

                subOrder.rankDesc = dealRank.get('description')

                subOrder.maxNlc = dealRank.get('maxNlc')

                subOrder.minNlc = dealRank.get('minNlc')

                subOrder.db = dealRank.get('dp')

                subOrder.itemStatus = dealRank.get('status')

                subOrders.append(subOrder)

            merchantOrder.subOrders = self.updateCashbackInSubOrders(subOrders)

            return merchantOrder

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp = {}

        try:

            rawHtmlSoup = BeautifulSoup(rawHtml)

            merchantOrder = self._parseUsingOrderJson(orderId, subTagId, userId, rawHtmlSoup, orderSuccessUrl)

            merchantOrder.orderTrackingUrl = ORDER_TRACK_URL%(long(merchantOrder.merchantOrderId))

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp

        except:

            print "Error occurred"

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            return resp

    def scrapeStoreOrders(self,):

        #collectionMap = {'palcedOn':1}

        searchMap = {}

        collectionMap = {"orderTrackingUrl":1,"merchantOrderId":1}

        orders = self._getActiveOrders(searchMap,collectionMap)

        for order in orders:

            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

            try:

                print "Order", self.store_name, order['orderId'], order['orderTrackingUrl'], order['merchantOrderId']

                br1 = track(HS_ORDER_TRACK_URL, order['merchantOrderId'])

                ungzipResponseBr(br1.response(), br1)

                trackPageSoup = BeautifulSoup(br1.response())

                subOrderTable = trackPageSoup.body.find("table", {'class':'lower-table'})

                subOrders = subOrderTable.find_all('tr')

                firstRow = subOrders.pop(0)

                closed = True

                for row in subOrders:

                    cols = row.find_all('td')

                    subOrderId = cols[0].text.strip()

                    subOrderStatus = cols[1].text.strip()

                    subbulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                    print 'Sub Order Id', str(subOrderId)

                    subOrder =  self._isSubOrderActive(order, str(subOrderId))

                    if subOrder is None:

                        print 'No HS Sub Order Found for SubOrder Id:- '+ str(subOrderId)

                    elif subOrder['closed']:

                        continue

                    else:

                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": str(subOrderId)}

                        updateMap = {}

                        displayStatus = self._getDisplayStatusFromDetailedStatus(subOrderStatus)

                        status = self._getStatusFromDetailedStatus(subOrderStatus) 

                        if displayStatus is not None:

                            updateMap["subOrders.$.detailedStatus"] = displayStatus

                        closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                        if status is not None:

                            updateMap["subOrders.$.status"] = status

                        if closedStatus:

                            #if status is closed then change the paybackStatus accordingly

                            print 'Order Closed'

                            updateMap["subOrders.$.closed"] = True

                            if status == Store.ORDER_DELIVERED:

                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                            elif status == Store.ORDER_CANCELLED:

                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                        else:

                            closed = False

                            print 'Order not Closed'

                        subbulk.find(findMap).update({'$set' : updateMap})

                        subresult = subbulk.execute()

                        tprint(subresult)

                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})

                result = bulk.execute()

                tprint(result)

            except:

                print "Error occurred for tracking order... "+str(order['merchantOrderId'])

                traceback.print_exc()

                continue

def track(url, orderId):

    br = getBrowserObject()

    br.set_proxies({"http": PROXY_MESH_GENERAL})

    response = br.open(url)

    ungzipResponseBr(response, br)

    soup = BeautifulSoup(br.response())

    csrf = soup.find('input', {'name': '_csrf'}).get('value')

    print csrf

    #html = response.read()

    #print html

    br.select_form(name='trackForm')

    br.form['orderId'] = str(orderId)

    response = br.submit()

    print "********************"

    print "Attempting to Login"

    print "********************"

    #ungzipResponse(response, br)

    return br

def ungzipResponseBr(r,b):

    headers = r.info()

    if headers['Content-Encoding']=='gzip':

        import gzip

        print "********************"

        print "Deflating gzip response"

        print "********************"

        gz = gzip.GzipFile(fileobj=r, mode='rb')

        html = gz.read()

        gz.close()

        headers["Content-type"] = "text/html; charset=utf-8"

        r.set_data( html )

        b.set_response(r)

def to_py_date(java_timestamp):

    date = datetime.fromtimestamp(java_timestamp)

    return date       

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj