WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/dtr/sources/snapdeal.py

 from dtr.api.Service import Orders
 from dtr.dao import AffiliateInfo, Order, SubOrder
 from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
     Store as MStore, ungzipResponse, tprint
 from dtr.storage import Mongo
-from dtr.storage.DataService import Order_Parse_Info, All_user_addresses
+from dtr.storage.DataService import Order_Parse_Info, All_user_addresses, \
+    OrdersRaw
 from dtr.storage.Mongo import getImgSrc, getDealRank
 from dtr.utils import utils
-from dtr.utils.utils import fetchResponseUsingProxy
+from dtr.utils.utils import fetchResponseUsingProxy, readSSh
 from elixir import *
 from pprint import pprint
 from pymongo import MongoClient
+from pyquery import PyQuery
 from urlparse import urlparse, parse_qs
 from xlrd import open_workbook
 import csv
 import json
 import os.path
     '''
     OrderStatusMap = {
                       MStore.ORDER_PLACED : ['in progress', 'pending for verification', 'not available', 'in process', 'processing', 'processed', 'under verification'],
                       MStore.ORDER_DELIVERED : ['delivered'],
                       MStore.ORDER_SHIPPED : ['in transit', 'dispatched'],
-                      MStore.ORDER_CANCELLED : ['closed for vendor reallocation', 'cancelled', 'product returned by courier', 'returned', 'n/a', 'courier returned', 'a new order placed with a different seller', 'closed']
+                      MStore.ORDER_CANCELLED : ['closed for vendor reallocation', 'cancelled', 'product returned by courier', 'returned', 'n/a', 'courier returned', 'a new order placed with a different seller', 'closed', 'cancellation in progress']
+                      }
     CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE
     def __init__(self,store_id):
         super(Store, self).__init__(store_id)
             subOrder.trackingUrl = trackingDet.span.a['href']
             subOrder.trackingNumber = trackingDet.span.a.text.strip()
         subOrders.append(subOrder)
         return subOrder
+    def getOrderJSON(self, rawHtml, supcMap):
+        #print rawHtml
+#        replace_with = {
+#        '<': '&gt;',
+#        '>': '&lt;',
+#        '&': '&amp;',
+#        '"': '&quot;', # should be escaped in attributes
+#        "'": '&apos;'    # should be escaped in attributes
+#        }
+        pq = PyQuery(rawHtml)
+        jsonValue = pq("#orderJSON").attr("value")
+        jsonValue.replace("&quot;", '"')
+        jsonValue.replace("&amp;", '&')
+        jsonValue.replace("&gt;", '>')
+        jsonValue.replace("&lt;", '<')
+        jsonValue.replace("&apos;", "'")
+        allSupcElements = pq('div.mdt-layout')('div.mdt-card')('div.order-item')
+        for supcElement in allSupcElements:
+            try:
+                supcElement = pq(supcElement)
+                title = supcElement('div.order-heading').text().strip()
+                productUrl = supcElement.attr("data-href")
+                imgUrl = supcElement.find('img').attr('src')
+                supc = imgUrl.split('-')[-3]
+                supcMap[supc] = {'title':title, 'imgUrl':imgUrl, 'productUrl':productUrl}
+            except:
+                pass
+        return json.loads(jsonValue)
     def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
                     #print merchantOrder
         resp = {}
+        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
+        supcMap = {}
         try:
-            br = getBrowserObject()
-            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
-            page = br.open(url)
-            page = ungzipResponse(page)
             try:
-                merchantOrder = self._parseB(orderId, subTagId, userId, page, orderSuccessUrl)
+                orderJSON = self.getOrderJSON(rawHtml, supcMap)
             except:
                 traceback.print_exc()
+                try:
+                    page = fetchResponseUsingProxy(orderSuccessUrl)
+                    orderJSON = self.getOrderJSON(page,supcMap)
+                except:
+                    traceback.print_exc()
+                    orderJSON = None
+            if orderJSON is None:
+                page =fetchResponseUsingProxy(url)
+                page = ungzipResponse(page)
+                try:
+                    merchantOrder = self._parseB(orderId, subTagId, userId, page, orderSuccessUrl)
+                except:
+                    traceback.print_exc()
-                merchantOrder = self._parse(orderId, subTagId, userId, page, orderSuccessUrl)
+                    merchantOrder = self._parse(orderId, subTagId, userId, page, orderSuccessUrl)
+            else:
+                merchantOrder = self._parseC(orderId, subTagId, userId, supcMap, orderJSON, orderSuccessUrl)
             merchantOrder.orderTrackingUrl = url
             if self._saveToOrder(todict(merchantOrder)):
                 resp['result'] = 'ORDER_CREATED'
             else:
                 resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
+            print "=================", resp, orderId, "=============="
             return resp
         except:
             print "Error occurred"
             traceback.print_exc()
             resp['result'] = 'ORDER_NOT_CREATED'
+            print "=================", resp, orderId, "=============="
             return resp
         #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)
         #soup.find(name, attrs, recursive, text)
+    def _parseC(self, orderId, subTagId, userId, supcMap, orderJSON, orderSuccessUrl):
+        print orderJSON
+        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
+        placedOn = datetime.strftime(utils.fromTimeStamp(orderJSON['created']/1000), "%a, %d %b, %Y")
+        merchantOrder.placedOn = placedOn
+        merchantOrder.merchantOrderId = orderJSON['code']
+        merchantOrder.paidAmount = orderJSON['paidAmount']
+        merchantOrder.deliveryCharges = orderJSON['shippingCharges']
+        merchantOrder.closed= False
+        merchantSubOrders = []
+        for s in orderJSON['suborders']:
+            print s
+            if not supcMap.has_key(s['supcCode']):
+                url = "http://www.snapdeal.com/search/autoSuggestion?q=%s&catId=0&ver=3"%s['supcCode']
+                html = utils.fetchResponseUsingProxy(url)
+                ul = PyQuery(html)('ul.top-products')
+                title = ul('div.product-text').text().strip()
+                productUrl = ul('a').attr("href").split("?")[0]
+                imgUrl = ul('img').attr('src')
+                supcMap[s['supcCode']] = {'title':title, 'imgUrl':imgUrl, 'productUrl':productUrl}
+            map1 = supcMap[s['supcCode']]
+            amountPaid = s['paidAmount']
+            productTitle = map1['title']
+            productUrl = map1['productUrl']
+            subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)
+            if(s.get('deliveryDate') is not None):
+                print "Delivered On",
+                subOrder.deliveredOn = datetime.strftime(utils.fromTimeStamp(s.get('deliveryDate')/1000),'%Y-%m-%d')
+                subOrder.status = MStore.ORDER_DELIVERED
+                subOrder.detailedStatus = MStore.ORDER_DELIVERED
+            elif s['suborderStatus'].get('macroDescription')== 'Closed':
+                if s['suborderStatus'].get('value')== 'Close for vendor reallocation':
+                    subOrder.detailedStatus = 'Close for vendor reallocation'
+                    subOrder.status = MStore.ORDER_CANCELLED
+            try:
+                subOrder.detailedStatus = s['suborderStatus']['macroDescription']
+                subOrder.status = self._getStatusFromDetailedStatus(subOrder.detailedStatus)
+            except:
+                print "----------------", s['suborderStatus']
+            subOrder.merchantSubOrderId = s['code']
+            subOrder.deliveryCharges = s['shippingCharges']
+            subOrder.productCode = s['supcCode']
+            subOrder.imgUrl = map1['imgUrl']
+            subOrder.unitPrice = s['offerPrice']  -s['internalCashbackValue'] - s['externalCashbackValue']
+            subOrder.amount = subOrder.unitPrice - s['offerDiscount'] - s['sdCash']
+            try:
+                try:
+                    if s['shipDateRange']['start']==s['shipDateRange']['end']:
+                        subOrder.estimatedShippingDate = datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['start']/1000),'%Y-%m-%d')
+                    else:
+                        subOrder.estimatedShippingDate = datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['start']/1000),'%Y-%m-%d') + " - " + datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['end']/1000),'%Y-%m-%d')
+                except:
+                    if s['deliveryDateRange']['start']==s['deliveryDateRange']['end']:
+                        subOrder.estimatedDeliveryDate = datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['start']/1000),'%Y-%m-%d')
+                    else:
+                        subOrder.estimatedDeliveryDate = datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['start']/1000),'%Y-%m-%d') + " - " + datetime.strftime('%Y-%m-%d',utils.fromTimeStamp(s['deliveryDateRange']['end']/1000),'%Y-%m-%d')
+            except:
+                pass
+            subOrder.offerDiscount = s['offerDiscount']
+            subOrder.unitPrice = s['offerPrice']
+            merchantSubOrders.append(subOrder)
+        merchantOrder.subOrders = merchantSubOrders
+        self.populateDerivedFields(merchantOrder)
+        return merchantOrder
     def _getStatusFromDetailedStatus(self, detailedStatus):
         for key, value in Store.OrderStatusMap.iteritems():
             if detailedStatus.lower() in value:
                 return key
 def main():
     #print todict([1,2,"3"])
     store = getStore(3)
-    store.scrapeStoreOrders()
+    #store.scrapeStoreOrders()
+    #https://m.snapdeal.com/purchaseMobileComplete?code=3fbc8a02a1c4d3c4e906f46886de0464&order=5808451506
+    #https://m.snapdeal.com/purchaseMobileComplete?code=9f4dfa49ff08a16d04c5e4bf519506fc&order=9611672826
+    orders = session.query(OrdersRaw).filter_by(store_id=3).filter_by(status='ORDER_NOT_CREATED').all()
+    for o in orders:
+        result = store.parseOrderRawHtml(o.id, o.sub_tag, o.user_id, o.rawhtml, o.order_url)['result']
+        o.status = result
+        session.commit()
-    #store.parseOrderRawHtml(332221, "3232311", 2, "32311243", "https://m.snapdeal.com/orderSummary?code=3fbc8a02a1c4d3c4e906f46886de0464&order=5808451506")
+        #store.parseOrderRawHtml(332221, "3232311", 2, readSSh("/home/amit/snapdeal.html"), "https://m.snapdeal.com/purchaseMobileComplete?code=3fbc8a02a1c4d3c4e906f46886de0464&order=5808451506")
     #store.scrapeStoreOrders()
     #store._isSubOrderActive(8, "5970688907")
     #store.scrapeAffiliate(datetime(2015,4,1))
     #store.scrapeStoreOrders()
     #store.parseInfo()

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/snapdeal.py – Rev 17237 → 17387