WebSVN – SmartDukaan – //trunk/PyDTR/src/dtr/sources/snapdeal.py

'''
Created on Jan 15, 2015

@author: amit
'''
from BeautifulSoup import BeautifulSoup
from bson.binary import Binary
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException
from pymongo import MongoClient
import datetime
import json
import pymongo
import re
import urllib
import traceback

from pprint import pprint
USERNAME='saholic1@gmail.com'
PASSWORD='spice@2020'
AFFILIATE_URL='http://affiliate.snapdeal.com'
POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'
CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'


class Store(main.Store):
    
    '''
    This is to map order statuses of our system to order statuses of snapdeal.
    And our statuses will change accordingly.
    
    '''
    OrderStatusMap = {
                      main.Store.ORDER_PLACED : ['In Progress','N/A'],
                      main.Store.ORDER_DELIVERED : ['Delivered'],
                      main.Store.ORDER_SHIPPED : ['In Transit'],
                      main.Store.ORDER_CANCELLED : ['Closed For Vendor Reallocation']
                      
                      }
    def __init__(self,store_id):
        client = MongoClient('mongodb://localhost:27017/')
        self.db = client.Dtr
        super(Store, self).__init__(store_id)

    def getName(self):
        return "snapdeal"
    
    def scrapeAffiliate(self, startDate=None, endDate=None):
        br = getBrowserObject()
        br.open(AFFILIATE_URL)
        br.select_form(nr=0)
        br.form['data[User][password]'] = PASSWORD 
        br.form['data[User][email]'] = USERNAME
        br.submit()
        response = br.open(CONFIG_URL)
        
        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response, br), re.IGNORECASE)[0]
        
        allOffers = self._getAllOffers(br, token)
        
        allPyOffers = [self.covertToObj(offer).__dict__ for offer in allOffers]
        self._saveToAffiliate(allPyOffers)
    
    
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        try:
            br = getBrowserObject()
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
            response = br.open(url)
            page = ungzipResponse(response, br)
            #page=page.decode("utf-8")
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            #orderHead = soup.find(name, attrs, recursive, text)
            sections = soup.findAll("section")
            
            #print sections
            
            order = sections[1]
            orderTrs = order.findAll("tr")
            
            placedOn = str(orderTrs[0].findAll("td")[1].text)
            
            #Pop two section elements
            sections.pop(0) 
            sections.pop(0)
            subOrders = sections
            
             
            merchantSubOrders = []
    
            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
            merchantOrder.merchantOderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]
            for orderTr in orderTrs:
                orderTrString = str(orderTr)
                if "Total Amount" in orderTrString:
                    merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]
                elif "Delivery Charges" in orderTrString:
                    merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]
                elif "Discount Applied" in orderTrString:
                    merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]
                elif "Paid Amount" in orderTrString:
                    merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]
    
            for subOrderElement in subOrders:
                productUrl = str(subOrderElement.find("a")['href'])
                subTable = subOrderElement.find("table", {"class":"lrPad"})
                subTrs = subTable.findAll("tr")
                unitPrice=None
                offerDiscount = None
                deliveryCharges = None
                amountPaid = None
                for subTr in subTrs:
                    subTrString = str(subTr)
                    if "Unit Price" in subTrString:
                        unitPrice = re.findall(r'\d+', subTrString)[0]
                    if "Quantity" in subTrString:
                        qty = re.findall(r'\d+', subTrString)[0]
                    elif "Offer Discount" in subTrString:
                        offerDiscount =   re.findall(r'\d+', subTrString)[0]
                    elif "Delivery Charges" in subTrString:
                        deliveryCharges =   re.findall(r'\d+', subTrString)[0]
                    elif "Subtotal" in subTrString:
                        if int(qty) > 0:
                            amountPaid =   str(int(re.findall(r'\d+', subTrString)[0])/int(qty))
                        else:
                            amountPaid =   "0"

                #TODO: cashbackAmount = getCashbackAmount()
                cashbackAmount = 0
                cashbackStatus = Store.CB_PENDING
                if cashbackAmount <= 0:
                    cashbackStatus = Store.CB_NA
                        
                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
                if len(divs)<=0:
                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
                
                for div in divs:
                    productTitle = str(subOrderElement.find("a").text)
                    productUrl = "http://m.snapdeal.com/" + productUrl 
                    subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)
    
                    subOrder.amountPaid = amountPaid
                    subOrder.deliveryCharges = deliveryCharges
                    subOrder.offerDiscount = offerDiscount
                    subOrder.unitPrice = unitPrice
                    subOrder.productCode = re.findall(r'\d+$', productUrl)[0]
                    subOrder.cashBackStatus = cashbackStatus
                    subOrder.cashBackAmount = cashbackAmount
                    
                    
                    trackAnchor = div.find("a")   
                    if trackAnchor is not None:
                        subOrder.tracingkUrl = str(trackAnchor['href'])
                    
                    divStr = str(div)
                    divStr = divStr.replace("\n","").replace("\t", "")
                    
                    for line in divStr.split("<br />"):
                        if "Suborder ID" in line:
                            subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]   
                        elif "Status" in line:
                            subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                        elif "Est. Shipping Date" in line:
                            subOrder.estimatedShippingDate = line.split(":")[1].strip()
                        elif "Est. Delivery Date" in line:
                            subOrder.estimatedDeliveryDate = line.split(":")[1].strip()
                        elif "Courier Name" in line:
                            subOrder.courierName = line.split(":")[1].strip()
                        elif "Tracking No" in line:
                            subOrder.trackingNumber = line.split(":")[1].strip()
                               
                merchantSubOrders.append(subOrder)   
            
            merchantOrder.subOrders = merchantSubOrders
            #print merchantOrder
            
            self._saveToOrder(todict(merchantOrder))
            return True
        except:
            print "Error occurred"
            traceback.print_exc()
        
        return False
        #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)
        #soup.find(name, attrs, recursive, text)
    
                      

    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, value in Store.OrderStatusMap.iteritems():
            if detailedStatus in value:
                return key
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
    
    def settlePayBack(self):
        pass
    
    def scrapeStoreOrders(self,):
        orders = self._getActiveOrders()
        br = getBrowserObject()
        for order in orders:
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
            response = br.open(url)
            page = ungzipResponse(response, br)
            #page=page.decode("utf-8")
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            sections = soup.findAll("section")
            sections.pop(0)
            sections.pop(0)
            
            subOrders = sections
            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
            for subOrderElement in subOrders:
                closed = True
                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
                if len(divs)<=0:
                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
                
                for div in divs:
                    divStr = str(div)
                    divStr = divStr.replace("\n","").replace("\t", "")
                    updateMap = {}
                    for line in divStr.split("<br />"):
                        if "Suborder ID" in line:
                            merchantSubOrderId = re.findall(r'\d+', line)[0]   
                            findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
                        elif "Status" in line:
                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                            updateMap["subOrders.$.detailedStatus"] = detailedStatus
                            status = self._getStatusFromDetailedStatus(detailedStatus) 
                            if closed:
                                closed = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
                        elif "Est. Shipping Date" in line:
                            estimatedShippingDate = line.split(":")[1].strip()
                            updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
                        elif "Est. Delivery Date" in line:
                            estimatedDeliveryDate = line.split(":")[1].strip()
                            updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
                        elif "Courier Name" in line:
                            courierName = line.split(":")[1].strip()
                            updateMap["subOrders.$.courierName"] = courierName
                        elif "Tracking No" in line:
                            trackingNumber = line.split(":")[1].strip()
                            updateMap["subOrders.$.trackingNumber"] = trackingNumber
                            
                    bulk.find(findMap).update({'$set' : updateMap})
                    bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed}})
            result = bulk.execute()
            pprint(result)        
            
    """
    This will insert records with changes only 
    """
    
    def _getActiveOrders(self):
        collection = self.db.merchantOrder
        collectionMap =  {"orderSuccessUrl":1, "orderId":1,"subOrders.merchantSubOrderId":1,"subOrders.closed":1}
        stores = collection.find({"closed": False, "storeId" : self.store_id}, collectionMap)
        return [store for store in stores]
        
        
        
    def _saveToAffiliate(self, offers):
        collection = self.db.snapdealOrderAffiliateInfo
        try:
            collection.insert(offers,continue_on_error=True)
        except pymongo.errors.DuplicateKeyError as e:
            print e.details
            
    def _saveToOrder(self, order):
        collection = self.db.merchantOrder
        try:
            order = collection.insert(order)
            #merchantOder 
        except Exception as e:
            traceback.print_exc()
    
    def _getAllOffers(self, br, token):
        allOffers = []
        nextPage = 1  
        while True:
            data = getPostData(token, nextPage)
            response = br.open(POST_URL, data)
            rmap = json.loads(ungzipResponse(response, br))
            if rmap is not None:
                rmap = rmap['response']
                if rmap is not None and len(rmap['errors'])==0:
                    allOffers += rmap['data']['data']
                    print allOffers
            nextPage += 1
            if rmap['data']['pageCount']<nextPage:
                break
        
        return allOffers
    
    def covertToObj(self,offer):
        offerData = offer['Stat']
        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 
                              offerData['datetime'], offerData['payout'], offer['Offer']['name'], offerData['ip'], offerData['conversion_sale_amount'])
        return offer1
def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):
    endDate=datetime.date.today() + datetime.timedelta(days=1)
    startDate=endDate - datetime.timedelta(days=31)

    parameters = (
        ("page",str(page)),
        ("limit",str(limit)),
        ("fields[]","Stat.offer_id"),
        ("fields[]","Stat.datetime"),
        ("fields[]","Offer.name"),
        ("fields[]","Stat.conversion_status"),
        ("fields[]","Stat.conversion_sale_amount"),
        ("fields[]","Stat.payout"),
        ("fields[]","Stat.ip"),
        ("fields[]","Stat.ad_id"),
        ("fields[]","Stat.affiliate_info1"),
        ("sort[Stat.datetime]","desc"),
        ("filters[Stat.date][conditional]","BETWEEN"),
        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),
        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),
        ("data_start",startDate.strftime('%Y-%m-%d')),
        ("data_end",endDate.strftime('%Y-%m-%d')),
        ("Method","getConversions"),
        ("NetworkId","jasper"),
        ("SessionToken",token),
    )
    #Encode the parameters
    return urllib.urlencode(parameters)

def main():
    store = getStore(3)
    store.scrapeStoreOrders()
    
    #store.parseOrderRawHtml(12345, "subtagId", 122323,  "html", 'https://m.snapdeal.com/purchaseMobileComplete?code=1f4166d13ea799b65aa9dea68b3e9e70&order=4509499363')

def ungzipResponse(r,b):
    headers = r.info()
    if headers['Content-Encoding']=='gzip':
        import gzip
        print "********************"
        print "Deflating gzip response"
        print "********************"
        gz = gzip.GzipFile(fileobj=r, mode='rb')
        html = gz.read()
        gz.close()
        return html

if __name__ == '__main__':
    main()

def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj
Subversion Repositories SmartDukaan

(root)//trunk/PyDTR/src/dtr/sources/snapdeal.py – Rev 13610
