WebSVN – SmartDukaan – //trunk/PyDTR/src/dtr/sources/snapdeal.py

'''
Created on Jan 15, 2015

@author: amit
'''
from BeautifulSoup import BeautifulSoup
from bson.binary import Binary
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException
from pymongo import MongoClient
import datetime
import json
import pymongo
import re
import urllib
from pprint import pprint
USERNAME='saholic1@gmail.com'
PASSWORD='spice@2020'
AFFILIATE_URL='http://affiliate.snapdeal.com'
POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'
CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'


class Store(main.Store):
    
    '''
    This is to map order statuses of our system to order statuses of snapdeal.
    And our statuses will change accordingly.
    
    '''
    OrderStatusMap = {
                      main.Store.ORDER_PLACED : ['In Progress','N/A'],
                      main.Store.ORDER_DELIVERED : ['Delivered'],
                      main.Store.ORDER_SHIPPED : ['In Transit'],
                      main.Store.ORDER_CANCELLED : ['Closed For Vendor Reallocation']
                      
                      }
    def __init__(self,store_id):
        client = MongoClient('mongodb://localhost:27017/')
        self.db = client.dtr
        super(Store, self).__init__(store_id)

    def getName(self):
        return "snapdeal"
    
    def scrapeAffiliate(self, startDate=None, endDate=None):
        br = getBrowserObject()
        br.open(AFFILIATE_URL)
        br.select_form(nr=0)
        br.form['data[User][password]'] = PASSWORD 
        br.form['data[User][email]'] = USERNAME
        br.submit()
        response = br.open(CONFIG_URL)
        
        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response, br), re.IGNORECASE)[0]
        
        allOffers = self._getAllOffers(br, token)
        
        allPyOffers = [self.covertToObj(offer).__dict__ for offer in allOffers]
        self._saveToAffiliate(allPyOffers)
    
    
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        br = getBrowserObject()
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
        response = br.open(url)
        page = ungzipResponse(response, br)
        #page=page.decode("utf-8")
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
        #orderHead = soup.find(name, attrs, recursive, text)
        sections = soup.findAll("section")
        
        #print sections
        
        order = sections[1]
        orderTrs = order.findAll("tr")
        
        placedOn = str(orderTrs[0].findAll("td")[1].text)
        
        #Pop two section elements
        sections.pop(0) 
        sections.pop(0)
        subOrders = sections
        
         
        merchantSubOrders = []

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
        merchantOrder.merchantOderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]
        for orderTr in orderTrs:
            orderTrString = str(orderTr)
            if "Total Amount" in orderTrString:
                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]
            elif "Delivery Charges" in orderTrString:
                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]
            elif "Discount Applied" in orderTrString:
                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]
            elif "Paid Amount" in orderTrString:
                merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]

        for subOrderElement in subOrders:
            productUrl = str(subOrderElement.find("a")['href'])
            subTable = subOrderElement.find("table", {"class":"lrPad"})
            subTrs = subTable.findAll("tr")
            unitPrice=None
            offerDiscount = None
            deliveryCharges = None
            amountPaid = None
            for subTr in subTrs:
                subTrString = str(subTr)
                if "Unit Price" in subTrString:
                    unitPrice = re.findall(r'\d+', subTrString)[0]
                if "Quantity" in subTrString:
                    qty = re.findall(r'\d+', subTrString)[0]
                elif "Offer Discount" in subTrString:
                    offerDiscount =   re.findall(r'\d+', subTrString)[0]
                elif "Delivery Charges" in subTrString:
                    deliveryCharges =   re.findall(r'\d+', subTrString)[0]
                elif "Subtotal" in subTrString:
                    amountPaid =   str(int(re.findall(r'\d+', subTrString)[0])/int(qty))
                    
            divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
            if len(divs)<=0:
                raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
            
            for div in divs:
                productTitle = str(subOrderElement.find("a").text)
                productUrl = "http://m.snapdeal.com/" + productUrl 
                subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

                subOrder.amountPaid = amountPaid
                subOrder.deliveryCharges = deliveryCharges
                subOrder.offerDiscount = offerDiscount
                subOrder.unitPrice = unitPrice
                subOrder.productCode = re.findall(r'\d+$', productUrl)[0]
                
                trackAnchor = div.find("a")   
                if trackAnchor is not None:
                    subOrder.tracingkUrl = str(trackAnchor['href'])
                
                divStr = str(div)
                divStr = divStr.replace("\n","").replace("\t", "")
                
                for line in divStr.split("<br />"):
                    if "Suborder ID" in line:
                        subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]   
                    elif "Status" in line:
                        subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                    elif "Est. Shipping Date" in line:
                        subOrder.estimatedShippingDate = line.split(":")[1].strip()
                    elif "Est. Delivery Date" in line:
                        subOrder.estimatedDeliveryDate = line.split(":")[1].strip()
                    elif "Courier Name" in line:
                        subOrder.courierName = line.split(":")[1].strip()
                    elif "Tracking No" in line:
                        subOrder.trackingNumber = line.split(":")[1].strip()
                           
            merchantSubOrders.append(subOrder)   
        
        merchantOrder.subOrders = merchantSubOrders
        #print merchantOrder
        
        self._saveToOrder(todict(merchantOrder))
        
        #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)
        #soup.find(name, attrs, recursive, text)
    
                      

    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, value in Store.OrderStatusMap.iteritems():
            if detailedStatus in value:
                return key
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
    
    def scrapeStoreOrders(self,):
        orders = self._getActiveOrders()
        br = getBrowserObject()
        for order in orders:
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
            response = br.open(url)
            page = ungzipResponse(response, br)
            #page=page.decode("utf-8")
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            sections = soup.findAll("section")
            sections.pop(0)
            sections.pop(0)
            
            subOrders = sections
            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
            for subOrderElement in subOrders:
                closed = True
                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
                if len(divs)<=0:
                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
                
                for div in divs:
                    trackAnchor = div.find("a")   
                    if trackAnchor is not None:
                        tracingkUrl = str(trackAnchor['href'])
                    
                    divStr = str(div)
                    divStr = divStr.replace("\n","").replace("\t", "")
                    updateMap = {}
                    for line in divStr.split("<br />"):
                        if "Suborder ID" in line:
                            merchantSubOrderId = re.findall(r'\d+', line)[0]   
                            findMap = {"id": order['id'], "subOrders.merchantSubOrderId": merchantSubOrderId}
                        elif "Status" in line:
                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                            updateMap["subOrders.$.detailedStatus"] = detailedStatus
                            status = self._getStatusFromDetailedStatus(detailedStatus) 
                            if closed:
                                closed = status in [Store.ORDER_PLACED, Store.ORDER_CANCELLED]
                        elif "Est. Shipping Date" in line:
                            estimatedShippingDate = line.split(":")[1].strip()
                            updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
                        elif "Est. Delivery Date" in line:
                            estimatedDeliveryDate = line.split(":")[1].strip()
                            updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
                        elif "Courier Name" in line:
                            courierName = line.split(":")[1].strip()
                            updateMap["subOrders.$.courierName"] = courierName
                        elif "Tracking No" in line:
                            trackingNumber = line.split(":")[1].strip()
                            updateMap["subOrders.$.trackingNumber"] = trackingNumber
                            
                    updateMap["subOrders.$.closed"] = closed
                    bulk.find(findMap).update({'$set' : updateMap})
            result = bulk.execute()
            pprint(result)        
            
    """
    This will insert records with changes only 
    """
    
    def _getActiveOrders(self):
        collection = self.db.merchantOrder
        stores = collection.find({"closed": False, "storeId" : self.store_id}, {"orderSuccessUrl":1, "id":1})
        return [store for store in stores]
        
        
        
    def _saveToAffiliate(self, offers):
        collection = self.db.snapdealOrderAffiliateInfo
        try:
            collection.insert(offers,continue_on_error=True)
        except pymongo.errors.DuplicateKeyError as e:
            print e.details
            
    def _saveToOrder(self, order):
        collection = self.db.merchantOrder
        try:
            order = collection.insert(order)
            #merchantOder 
        except Exception as e:
            print todict(e)
    
    def _getAllOffers(self, br, token):
        allOffers = []
        nextPage = 1  
        while True:
            data = getPostData(token, nextPage)
            response = br.open(POST_URL, data)
            rmap = json.loads(ungzipResponse(response, br))
            if rmap is not None:
                rmap = rmap['response']
                if rmap is not None and len(rmap['errors'])==0:
                    allOffers += rmap['data']['data']
                    print allOffers
            nextPage += 1
            if rmap['data']['pageCount']<nextPage:
                break
        
        return allOffers
    
    def covertToObj(self,offer):
        offerData = offer['Stat']
        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 
                              offerData['datetime'], offerData['payout'], offer['Offer']['name'], offerData['ip'], offerData['conversion_sale_amount'])
        return offer1
def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):
    endDate=datetime.date.today() + datetime.timedelta(days=1)
    startDate=endDate - datetime.timedelta(days=31)

    parameters = (
        ("page",str(page)),
        ("limit",str(limit)),
        ("fields[]","Stat.offer_id"),
        ("fields[]","Stat.datetime"),
        ("fields[]","Offer.name"),
        ("fields[]","Stat.conversion_status"),
        ("fields[]","Stat.conversion_sale_amount"),
        ("fields[]","Stat.payout"),
        ("fields[]","Stat.ip"),
        ("fields[]","Stat.ad_id"),
        ("fields[]","Stat.affiliate_info1"),
        ("sort[Stat.datetime]","desc"),
        ("filters[Stat.date][conditional]","BETWEEN"),
        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),
        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),
        ("data_start",startDate.strftime('%Y-%m-%d')),
        ("data_end",endDate.strftime('%Y-%m-%d')),
        ("Method","getConversions"),
        ("NetworkId","jasper"),
        ("SessionToken",token),
    )
    #Encode the parameters
    return urllib.urlencode(parameters)

def main():
    store = getStore(3)
    store.scrapeStoreOrders()
    #store.scrapeAffiliate()
    #with open ("data.txt", "r") as myfile:
    #    data=myfile.read()
    #    myfile.close()
    
    #store.parseOrderRawHtml(12345, "subtagId", 122323,  "html", 'https://m.snapdeal.com/purchaseMobileComplete?code=1f4166d13ea799b65aa9dea68b3e9e70&order=4509499363')

def ungzipResponse(r,b):
    headers = r.info()
    if headers['Content-Encoding']=='gzip':
        import gzip
        print "********************"
        print "Deflating gzip response"
        print "********************"
        gz = gzip.GzipFile(fileobj=r, mode='rb')
        html = gz.read()
        gz.close()
        return html

if __name__ == '__main__':
    main()

def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj
Subversion Repositories SmartDukaan

(root)//trunk/PyDTR/src/dtr/sources/snapdeal.py – Rev 13576
