WebSVN – SmartDukaan – Diff – //trunk/PyDTR/src/dtr/sources/snapdeal.py

 '''
 Created on Jan 15, 2015
 @author: amit
 '''
+from BeautifulSoup import BeautifulSoup
 from bson.binary import Binary
 from dtr import main
-from dtr.dao import AffiliateInfo
+from dtr.dao import AffiliateInfo, Order, SubOrder
 from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException
 from pymongo import MongoClient
-from  BeautifulSoup import BeautifulSoup
 import datetime
 import json
-import mechanize
 import pymongo
 import re
 import urllib
-import soupselect;soupselect.monkeypatch()
+from pprint import pprint
 USERNAME='saholic1@gmail.com'
 PASSWORD='spice@2020'
 AFFILIATE_URL='http://affiliate.snapdeal.com'
 POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
 ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'
                       main.Store.ORDER_SHIPPED : ['In Transit'],
                       main.Store.ORDER_CANCELLED : ['Closed For Vendor Reallocation']
+                      }
     def __init__(self,store_id):
+        client = MongoClient('mongodb://localhost:27017/')
+        self.db = client.dtr
         super(Store, self).__init__(store_id)
     def getName(self):
         return "snapdeal"
         allPyOffers = [self.covertToObj(offer).__dict__ for offer in allOffers]
         self._saveToAffiliate(allPyOffers)
-    def parseOrderPage(self, htmlString=None):
-        raise NotImplementedError
-    def parseOrderRawHtml(self, orderId, subtagId, userId, rawHtml, orderSuccessUrl):
+    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
         br = getBrowserObject()
         url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
-        print url
         response = br.open(url)
         page = ungzipResponse(response, br)
         #page=page.decode("utf-8")
         soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
         #orderHead = soup.find(name, attrs, recursive, text)
         #Pop two section elements
         sections.pop(0)
         sections.pop(0)
         subOrders = sections
-        merchantOrder = {}
         merchantSubOrders = []
-        merchantOrder["_id"] = orderId
-        merchantOrder["userId"] = userId
-        merchantOrder['subTagId'] = subtagId
+        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
-        merchantOrder['merchantOderId'] = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]
+        merchantOrder.merchantOderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]
-        merchantOrder['placedOn'] = placedOn
         for orderTr in orderTrs:
             orderTrString = str(orderTr)
             if "Total Amount" in orderTrString:
-                merchantOrder['totalAmount'] = re.findall(r'\d+', orderTrString)[0]
+                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]
             elif "Delivery Charges" in orderTrString:
-                merchantOrder['deliveryCharges'] = re.findall(r'\d+', orderTrString)[0]
+                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]
             elif "Discount Applied" in orderTrString:
-                merchantOrder['discountApplied'] = re.findall(r'\d+', orderTrString)[0]
+                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]
             elif "Paid Amount" in orderTrString:
-                merchantOrder['paidAmount'] = re.findall(r'\d+', orderTrString)[0]
+                merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]
-        merchantOrder['closed'] = False
         for subOrderElement in subOrders:
             productUrl = str(subOrderElement.find("a")['href'])
             subTable = subOrderElement.find("table", {"class":"lrPad"})
             subTrs = subTable.findAll("tr")
             unitPrice=None
             divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
             if len(divs)<=0:
                 raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
             for div in divs:
-                merchantSubOrder = {}
-                merchantSubOrder['placedOn'] = placedOn
-                merchantSubOrder['productTitle'] = str(subOrderElement.find("a").text)
+                productTitle = str(subOrderElement.find("a").text)
-                merchantSubOrder['productUrl'] = "http://m.snapdeal.com/" + productUrl
+                productUrl = "http://m.snapdeal.com/" + productUrl
-                merchantSubOrder['productCode'] = re.findall(r'\d+$', productUrl)[0]
+                subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)
-                merchantSubOrder['quantity'] =   1
-                merchantSubOrder['status'] = 'Order Placed'
-                merchantSubOrder['amountPaid'] = amountPaid
+                subOrder.amountPaid = amountPaid
-                merchantSubOrder['deliveryCharges'] = deliveryCharges
+                subOrder.deliveryCharges = deliveryCharges
-                merchantSubOrder['offerDiscount'] = offerDiscount
+                subOrder.offerDiscount = offerDiscount
-                merchantSubOrder['unitPrice'] = unitPrice
+                subOrder.unitPrice = unitPrice
+                subOrder.productCode = re.findall(r'\d+$', productUrl)[0]
                 trackAnchor = div.find("a")
                 if trackAnchor is not None:
-                    merchantSubOrder['tracingkUrl'] = str(trackAnchor['href'])
+                    subOrder.tracingkUrl = str(trackAnchor['href'])
                 divStr = str(div)
                 divStr = divStr.replace("\n","").replace("\t", "")
                 for line in divStr.split("<br />"):
                     if "Suborder ID" in line:
-                        merchantSubOrder['merchantSubOrderId'] = re.findall(r'\d+', line)[0]
+                        subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]
                     elif "Status" in line:
-                        merchantSubOrder['detailedStatus'] = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
+                        subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                     elif "Est. Shipping Date" in line:
-                        merchantSubOrder['estimatedShippingDate'] = line.split(":")[1].strip()
+                        subOrder.estimatedShippingDate = line.split(":")[1].strip()
                     elif "Est. Delivery Date" in line:
-                        merchantSubOrder['estimatedDeliveryDate'] = line.split(":")[1].strip()
+                        subOrder.estimatedDeliveryDate = line.split(":")[1].strip()
                     elif "Courier Name" in line:
-                        merchantSubOrder['courierName'] = line.split(":")[1].strip()
+                        subOrder.courierName = line.split(":")[1].strip()
                     elif "Tracking No" in line:
-                        merchantSubOrder['trackingNumber'] = line.split(":")[1].strip()
+                        subOrder.trackingNumber = line.split(":")[1].strip()
-            merchantSubOrders.append(merchantSubOrder)
+            merchantSubOrders.append(subOrder)
+        merchantOrder.subOrders = merchantSubOrders
+        #print merchantOrder
+        self._saveToOrder(todict(merchantOrder))
-        merchantOrder['subOrders'] = merchantSubOrders
-        print merchantOrder
         #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)
         #soup.find(name, attrs, recursive, text)
+    def _getStatusFromDetailedStatus(self, detailedStatus):
+        for key, value in Store.OrderStatusMap.iteritems():
+            if detailedStatus in value:
+                return key
+        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
     def scrapeStoreOrders(self,):
+        orders = self._getActiveOrders()
+        br = getBrowserObject()
+        for order in orders:
+            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
+            response = br.open(url)
+            page = ungzipResponse(response, br)
+            #page=page.decode("utf-8")
+            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
+            sections = soup.findAll("section")
-        pass
+            sections.pop(0)
+            sections.pop(0)
+            subOrders = sections
+            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
+            for subOrderElement in subOrders:
+                closed = True
+                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
+                if len(divs)<=0:
+                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
+                for div in divs:
+                    trackAnchor = div.find("a")
+                    if trackAnchor is not None:
+                        tracingkUrl = str(trackAnchor['href'])
+                    divStr = str(div)
+                    divStr = divStr.replace("\n","").replace("\t", "")
+                    updateMap = {}
+                    for line in divStr.split("<br />"):
+                        if "Suborder ID" in line:
+                            merchantSubOrderId = re.findall(r'\d+', line)[0]
+                            findMap = {"id": order['id'], "subOrders.merchantSubOrderId": merchantSubOrderId}
+                        elif "Status" in line:
+                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
+                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
+                            updateMap["subOrders.$.detailedStatus"] = detailedStatus
+                            status = self._getStatusFromDetailedStatus(detailedStatus)
+                            if closed:
+                                closed = status in [Store.ORDER_PLACED, Store.ORDER_CANCELLED]
+                        elif "Est. Shipping Date" in line:
+                            estimatedShippingDate = line.split(":")[1].strip()
+                            updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
+                        elif "Est. Delivery Date" in line:
+                            estimatedDeliveryDate = line.split(":")[1].strip()
+                            updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
+                        elif "Courier Name" in line:
+                            courierName = line.split(":")[1].strip()
+                            updateMap["subOrders.$.courierName"] = courierName
+                        elif "Tracking No" in line:
+                            trackingNumber = line.split(":")[1].strip()
+                            updateMap["subOrders.$.trackingNumber"] = trackingNumber
+                    updateMap["subOrders.$.closed"] = closed
+                    bulk.find(findMap).update({'$set' : updateMap})
+            result = bulk.execute()
+            pprint(result)
     """
     This will insert records with changes only
     """
+    def _getActiveOrders(self):
+        collection = self.db.merchantOrder
+        stores = collection.find({"closed": False, "storeId" : self.store_id}, {"orderSuccessUrl":1, "id":1})
+        return [store for store in stores]
     def _saveToAffiliate(self, offers):
-        client = MongoClient('mongodb://localhost:27017/')
-        db = client.dtr
-        collection = db.snapdealOrderAffiliateInfo
+        collection = self.db.snapdealOrderAffiliateInfo
         try:
             collection.insert(offers,continue_on_error=True)
         except pymongo.errors.DuplicateKeyError as e:
             print e.details
+    def _saveToOrder(self, order):
+        collection = self.db.merchantOrder
+        try:
+            order = collection.insert(order)
+            #merchantOder
+        except Exception as e:
+            print todict(e)
     def _getAllOffers(self, br, token):
         allOffers = []
         nextPage = 1
         while True:
+    )
     #Encode the parameters
     return urllib.urlencode(parameters)
 def main():
-    print "hello"
     store = getStore(3)
+    store.scrapeStoreOrders()
     #store.scrapeAffiliate()
     #with open ("data.txt", "r") as myfile:
     #    data=myfile.read()
     #    myfile.close()
-    store.parseOrderRawHtml(12345, "subtagId", 122323,  "html", 'https://m.snapdeal.com/purchaseMobileComplete?code=1f4166d13ea799b65aa9dea68b3e9e70&order=4509499363')
+    #store.parseOrderRawHtml(12345, "subtagId", 122323,  "html", 'https://m.snapdeal.com/purchaseMobileComplete?code=1f4166d13ea799b65aa9dea68b3e9e70&order=4509499363')
 def ungzipResponse(r,b):
     headers = r.info()
     if headers['Content-Encoding']=='gzip':
         import gzip
         gz.close()
         return html
 if __name__ == '__main__':
     main()
+def todict(obj, classkey=None):
+    if isinstance(obj, dict):
+        data = {}
+        for (k, v) in obj.items():
+            data[k] = todict(v, classkey)
+        return data
+    elif hasattr(obj, "_ast"):
+        return todict(obj._ast())
+    elif hasattr(obj, "__iter__"):
+        return [todict(v, classkey) for v in obj]
+    elif hasattr(obj, "__dict__"):
+        data = dict([(key, todict(value, classkey))
+            for key, value in obj.__dict__.iteritems()
+            if not callable(value) and not key.startswith('_')])
+        if classkey is not None and hasattr(obj, "__class__"):
+            data[classkey] = obj.__class__.__name__
+        return data
+    else:
+        return obj

Subversion Repositories SmartDukaan

(root)//trunk/PyDTR/src/dtr/sources/snapdeal.py – Rev 13569 → 13576