WebSVN – SmartDukaan – /trunk/PyDTR/src/dtr/sources/homeshop18.py

'''
Created on Jan 15, 2015

@author: Manish 
'''
from bs4 import BeautifulSoup
from bson.binary import Binary
from datetime import datetime, date, timedelta
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder, HomeShopAffiliateInfo
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
    Store as MStore, ungzipResponse, tprint
from dtr.storage import Mongo
from dtr.storage.Mongo import getImgSrc
from dtr.utils.utils import fetchResponseUsingProxy, PROXY_MESH_GENERAL
from pprint import pprint
from pymongo import MongoClient
import json
import pymongo
import re
import time
import traceback
import urllib
import urllib2
from urlparse import urlparse, parse_qs
import xml.etree.ElementTree as ET
from dtr.storage import MemCache
from dtr.storage.Mongo import getDealRank


AFFLIATE_TRASACTIONS_URL = "https://admin.optimisemedia.com/v2/reports/affiliate/leads/leadsummaryexport.aspx?Contact=796881&Country=26&Agency=95&Merchant=331902&Status=-1&Year=%d&Month=%d&Day=%d&EndYear=%d&EndMonth=%d&EndDay=%d&DateType=0&Sort=CompletionDate&Login=1347562DA5E3EFF6FB1561765C47C782&Format=XML&RestrictURL=0"
ORDER_TRACK_URL='https://m.homeshop18.com/order/orderDetail.mobi?orderId=%d'
HS_ORDER_TRACK_URL='http://www.homeshop18.com/track-your-order.html'
BASE_URL= 'http://www.shopclues.com'
BASE_MURL= 'http://m.shopclues.com'
BASE_PRODUCT_URL= 'http://m.homeshop18.com/product.mobi?productId=%d'
BASE_IMG_URL='http://stat.homeshop18.com/homeshop18'


#http://m.homeshop18.com/checkout/paySuccess.mobi?orderComplete=true

class Store(MStore):
    '''
    This is to map order statuses of our system to order statuses of snapdeal.
    And our statuses will change accordingly.
    
    '''
    OrderStatusMap = {
                      MStore.ORDER_PLACED : ['payment successful', 'new order - cod confirmation pending', 'processing', 'quality check','on schedule', 'processing - pickup initiated', 'processing - ready to dispatch','processing - procurement delay from merchant','processing - slight procurment delay from merchant','cod order confirmed by customer'],
                      MStore.ORDER_DELIVERED : ['delivered', 'complete'],
                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched','shipped','order handed to courier','order handed over to courier'],
                      MStore.ORDER_CANCELLED : ['payment failed', 'canceled', 'payment declined', 'order on hold - cancellation requested by customer', 'courier returned', 'canceled on customer request', 'canceled by customer','order canceled by customer','canceled - address not shippable','return complete','undelivered - returning to origin']
                      }
    OrderStatusConfirmationMap= {
                                 "P" : "Payment Successful",
                                 "D" : "Order Declined",
                                 "O" : "New Order - COD confirmation Pending"
                                 }
    
    OrderStatusStringMap = {
                            MStore.ORDER_PLACED : ['expect the order to reach', 'received your payment'],
                            MStore.ORDER_DELIVERED : ['has been delivered'],
                            MStore.ORDER_SHIPPED : ['has been shipped', 'has been dispatched'], 
                            MStore.ORDER_CANCELLED : ['has been cancelled', 'has been rejected','is returned back to us','payment failed']
                            }
    
    OrderStatusShownMap = {
                            "Under Process" : ['expect the order to reach', 'received your payment'],
                            "Order Delivered" : ['has been delivered'],
                            "Order Shipped" : ['has been shipped', 'has been dispatched'], 
                            "Order Cancelled" : ['has been cancelled', 'has been rejected','is returned back to us','payment failed']
                           }
    
    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE
    
    
    def __init__(self,store_id):
        super(Store, self).__init__(store_id)
    
    def convertToObj(self,offer):
        orderRef = offer['MerchantRef']
        if len(orderRef)>15:
            orderRef = orderRef[0:len(orderRef)-10]
        offer1 = HomeShopAffiliateInfo(offer['UID'], offer['TransactionTime'], offer['TransactionID'], orderRef,  orderRef, offer['Merchant'], offer['PID'], offer['Product'], float(str(offer['SR'])), float(str(offer['TransactionValue'])), offer['UKey'], offer['ClickTime'], offer['Status'])
        return offer1
    
    def _saveToAffiliate(self, offers):
        collection = self.db.homeshopOrderAffiliateInfo
        mcollection = self.db.merchantOrder
        for offerObj in offers:
            offer = self.convertToObj(offerObj)
            collection.update({"transactionId":offer.transactionId, "subTagId":offer.subTagId, "payOut":offer.payOut},{"$set":todict(offer)}, upsert=True)
            mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True}, {"$set":{"subOrders.$.missingAff":False}})
    
    def scrapeAffiliate(self, startDate=datetime.today() - timedelta(days=10), endDate=datetime.today()):
        uri = AFFLIATE_TRASACTIONS_URL%(startDate.year,startDate.month,startDate.day,endDate.year,endDate.month,endDate.day)
        root = ET.parse(urllib2.urlopen(uri)).getroot()
        if len(root)> 0 and len(root[0])> 0:
            offers = []
            for child in root[0][0]:
                offers.append(child.attrib)
            self._saveToAffiliate(offers)
            
    def _setLastSaleDate(self, saleDate):
        self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})
    
    def getName(self):
        return "homeshop18"  
    
    
    def _getLastSaleDate(self,):
        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})
        if lastDaySaleObj is None:
            return datetime.min
        
    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, statusList in Store.OrderStatusStringMap.iteritems():
            for value in statusList:
                if value in detailedStatus.lower():
                    return key
        print "Detailed Status need to be mapped", detailedStatus, self.store_id
        return None
    
    def _getDisplayStatusFromDetailedStatus(self, detailedStatus):
        for key, statusList in Store.OrderStatusShownMap.iteritems():
            for value in statusList:
                if value in detailedStatus.lower():
                    return key
        print "Display Status need to be mapped", detailedStatus, self.store_id
        return None
    
    def updateCashbackInSubOrders(self, subOrders):
        for subOrder in subOrders:
            cashbackStatus = Store.CB_NA
            cashbackAmount = 0
            percentage = 0
            amount = subOrder.amountPaid
            if amount > 0:
                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)
                if cashbackAmount > 0:
                    cashbackStatus = Store.CB_PENDING
            subOrder.cashBackStatus = cashbackStatus
            subOrder.cashBackAmount = cashbackAmount
            subOrder.cashBackPercentage = percentage
        return subOrders

    def _parseUsingOrderJson(self, orderId, subTagId, userId, rawHtmlSoup, orderSuccessUrl):
        orderObj = None
        
        scripts = rawHtmlSoup.find_all('script')
        for script in scripts:
            if 'var order =' in script.text:
                requiredObjList = script.text.strip().split('\n')
                for val in requiredObjList:
                    if "$.parseJSON('" in val:
                        val = val.encode("ascii","ignore")
                        print 'Val......',val
                        print val.split("$.parseJSON('")[1].split("');")[0]
                        orderObj = json.loads(val.split("$.parseJSON('")[1].split("');")[0])
                        print orderObj
                        break
                break
        
        if orderObj is not None:
            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
            merchantOrder.placedOn = orderObj['orderDate']
            merchantOrder.merchantOrderId = str(long(orderObj['orderId']))
            merchantOrder.paidAmount = long(orderObj['pricing']['orderNetPrice'])
            merchantOrder.totalAmount = long(orderObj['pricing']['orderGrossPrice'])
            merchantOrder.discountApplied = long(orderObj['pricing']['discountCouponRedemptionAmount'])+long(orderObj['pricing']['giftCouponRedemptionAmount'])
            merchantOrder.deliveryCharges = long(orderObj['totalShipmentCharges'])
            subOrders= []
            for subOrderObj in orderObj['subOrders']:
                subOrder = SubOrder(subOrderObj['cartItem']['cartItemTitle'], BASE_PRODUCT_URL%(long(subOrderObj['cartItem']['productId'])), orderObj['orderDate'], long(subOrderObj['pricing']['payablePrice']))
                subOrder.estimatedDeliveryDate = subOrderObj['shipment']['expectedDeliveryDate']
                subOrder.merchantSubOrderId = str(subOrderObj['subOrderId'])
                if rawHtmlSoup.body.find("div", {'class':'sub-order-status'}) is not None:
                    subOrder.detailedStatus = rawHtmlSoup.body.find("div", {'class':'sub-order-status'}).text
                else:
                    subOrder.detailedStatus = 'Order Placed'
                subOrder.imgUrl = BASE_IMG_URL+subOrderObj['cartItem']['lineItemImageUrl']
                subOrder.offerDiscount = (long(subOrderObj['cartItem']['price'])+long(subOrderObj['shipment']['shipmentCharge']))*long(subOrderObj['cartItem']['itemQuantity'])-long(subOrderObj['pricing']['payablePrice'])
                subOrder.unitPrice = long(subOrderObj['cartItem']['price'])
                subOrder.productCode = str(long(subOrderObj['cartItem']['productId']))
                subOrder.amountPaid = long(subOrderObj['pricing']['payablePrice'])
                subOrder.quantity = long(subOrderObj['cartItem']['itemQuantity'])
                subOrder.tracingkUrl = ORDER_TRACK_URL%(long(orderObj['orderId']))
                dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)
                subOrder.dealRank = dealRank.get('rank')
                subOrder.rankDesc = dealRank.get('description')
                subOrder.maxNlc = dealRank.get('maxNlc')
                subOrder.minNlc = dealRank.get('minNlc')
                subOrder.db = dealRank.get('dp')
                subOrder.itemStatus = dealRank.get('status')
                subOrders.append(subOrder)
            merchantOrder.subOrders = self.updateCashbackInSubOrders(subOrders)
            return merchantOrder
        
    
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        resp = {}
        try:
            rawHtmlSoup = BeautifulSoup(rawHtml)
            merchantOrder = self._parseUsingOrderJson(orderId, subTagId, userId, rawHtmlSoup, orderSuccessUrl)
            merchantOrder.orderTrackingUrl = ORDER_TRACK_URL%(long(merchantOrder.merchantOrderId))
            if self._saveToOrder(todict(merchantOrder)):
                resp['result'] = 'ORDER_CREATED'
            else:
                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                
            return resp
        except:
            print "Error occurred"
            traceback.print_exc()
            resp['result'] = 'ORDER_NOT_CREATED'
            return resp
        
    def scrapeStoreOrders(self,):
        #collectionMap = {'palcedOn':1}
        searchMap = {}
        collectionMap = {"orderTrackingUrl":1,"merchantOrderId":1}
        orders = self._getActiveOrders(searchMap,collectionMap)
        for order in orders:
            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
            try:
                print "Order", self.store_name, order['orderId'], order['orderTrackingUrl'], order['merchantOrderId']
                br1 = track(HS_ORDER_TRACK_URL, order['merchantOrderId'])
                ungzipResponseBr(br1.response(), br1)
                trackPageSoup = BeautifulSoup(br1.response())
                subOrderTable = trackPageSoup.body.find("table", {'class':'lower-table'})
                subOrders = subOrderTable.find_all('tr')
                firstRow = subOrders.pop(0)
                closed = True
                for row in subOrders:
                    cols = row.find_all('td')
                    subOrderId = cols[0].text.strip()
                    subOrderStatus = cols[1].text.strip()
                    subbulk = self.db.merchantOrder.initialize_ordered_bulk_op()
                    print 'Sub Order Id', str(subOrderId)
                    subOrder =  self._isSubOrderActive(order, str(subOrderId))
                    if subOrder is None:
                        print 'No HS Sub Order Found for SubOrder Id:- '+ str(subOrderId)
                    elif subOrder['closed']:
                        continue
                    else:
                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": str(subOrderId)}
                        updateMap = {}
                        displayStatus = self._getDisplayStatusFromDetailedStatus(subOrderStatus)
                        status = self._getStatusFromDetailedStatus(subOrderStatus) 
                        if displayStatus is not None:
                            updateMap["subOrders.$.detailedStatus"] = displayStatus
                        closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
                        if status is not None:
                            updateMap["subOrders.$.status"] = status
                        if closedStatus:
                            #if status is closed then change the paybackStatus accordingly
                            print 'Order Closed'
                            updateMap["subOrders.$.closed"] = True
                            if status == Store.ORDER_DELIVERED:
                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
                            elif status == Store.ORDER_CANCELLED:
                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
                        else:
                            closed = False
                            print 'Order not Closed'
                            
                        subbulk.find(findMap).update({'$set' : updateMap})
                        subresult = subbulk.execute()
                        tprint(subresult)
                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
                result = bulk.execute()
                tprint(result)
            except:
                print "Error occurred for tracking order... "+str(order['merchantOrderId'])
                traceback.print_exc()
                continue
                
            
            
def track(url, orderId):
    br = getBrowserObject()
    br.set_proxies({"http": PROXY_MESH_GENERAL})
    response = br.open(url)
    ungzipResponseBr(response, br)
    soup = BeautifulSoup(br.response())
    csrf = soup.find('input', {'name': '_csrf'}).get('value')
    print csrf
    #html = response.read()
    #print html
    br.select_form(name='trackForm')
    br.form['orderId'] = str(orderId)
    response = br.submit()
    print "********************"
    print "Attempting to Login"
    print "********************"
    #ungzipResponse(response, br)
    return br

def ungzipResponseBr(r,b):
    headers = r.info()
    if headers['Content-Encoding']=='gzip':
        import gzip
        print "********************"
        print "Deflating gzip response"
        print "********************"
        gz = gzip.GzipFile(fileobj=r, mode='rb')
        html = gz.read()
        gz.close()
        headers["Content-type"] = "text/html; charset=utf-8"
        r.set_data( html )
        b.set_response(r)
            

def to_py_date(java_timestamp):
    date = datetime.fromtimestamp(java_timestamp)
    return date       
        
def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj
Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/homeshop18.py @ 21776 – Rev 17309
