WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/dtr/sources/snapdeal.py

     This is to map order statuses of our system to order statuses of snapdeal.
     And our statuses will change accordingly.
     '''
     OrderStatusMap = {
-                      MStore.ORDER_PLACED : ['in progress', 'pending for verification', 'not available', 'in process', 'processing', 'processed', 'under verification'],
+                      MStore.ORDER_PLACED : ['in progress', 'pending for verification', 'not available', 'in process',
+                                             'processing', 'processed', 'under verification', 'readying for dispatch',
+                                             'waiting for courier to pick up', ''],
-                      MStore.ORDER_DELIVERED : ['delivered'],
+                      MStore.ORDER_DELIVERED : ['delivered','delivered successfully!'],
-                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched'],
+                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched', 'handed over to courier', 'undelivered. update delivery details!'],
-                      MStore.ORDER_CANCELLED : ['closed for vendor reallocation', 'cancelled', 'product returned by courier', 'returned', 'n/a', 'courier returned', 'a new order placed with a different seller', 'closed', 'cancellation in progress']
+                      MStore.ORDER_CANCELLED : ['closed for vendor reallocation', 'cancelled', 'product returned by courier', 'returned', 'n/a', 'courier returned',
+                                                'a new order placed with a different seller', 'closed', 'cancellation in progress', 'verification failed. order cancelled',
+                                                'cancelled. payment refunded','closed. new order placed']
+                      }
     CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE
     def __init__(self,store_id):
         super(Store, self).__init__(store_id)
             productTitle = map1['title']
             productUrl = map1['productUrl']
             subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)
             if(s.get('deliveryDate') is not None):
                 print "Delivered On",
-                subOrder.deliveredOn = datetime.strftime(utils.fromTimeStamp(s.get('deliveryDate')/1000),'%Y-%m-%d')
+                subOrder.deliveredOn = datetime.strftime(utils.fromTimeStamp(s.get('deliveryDate')/1000),'%d %b, %Y')
                 subOrder.status = MStore.ORDER_DELIVERED
                 subOrder.detailedStatus = MStore.ORDER_DELIVERED
             elif s['suborderStatus'].get('macroDescription')== 'Closed':
                 if s['suborderStatus'].get('value')== 'Close for vendor reallocation':
                     subOrder.detailedStatus = 'Close for vendor reallocation'
             subOrder.unitPrice = s['offerPrice']  -s['internalCashbackValue'] - s['externalCashbackValue']
             subOrder.amount = subOrder.unitPrice - s['offerDiscount'] - s['sdCash']
             try:
                 try:
                     if s['shipDateRange']['start']==s['shipDateRange']['end']:
-                        subOrder.estimatedShippingDate = datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['start']/1000),'%Y-%m-%d')
+                        subOrder.estimatedShippingDate = datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['start']/1000),'%d %b, %Y')
                     else:
-                        subOrder.estimatedShippingDate = datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['start']/1000),'%Y-%m-%d') + " - " + datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['end']/1000),'%Y-%m-%d')
+                        subOrder.estimatedShippingDate = datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['start']/1000),'%d %b, %Y') + " - " + datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['end']/1000),'%d %b, %Y')
                 except:
                     if s['deliveryDateRange']['start']==s['deliveryDateRange']['end']:
-                        subOrder.estimatedDeliveryDate = datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['start']/1000),'%Y-%m-%d')
+                        subOrder.estimatedDeliveryDate = datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['start']/1000),'%d %b, %Y')
                     else:
-                        subOrder.estimatedDeliveryDate = datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['start']/1000),'%Y-%m-%d') + " - " + datetime.strftime('%Y-%m-%d',utils.fromTimeStamp(s['deliveryDateRange']['end']/1000),'%Y-%m-%d')
+                        subOrder.estimatedDeliveryDate = datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['start']/1000),'%d %b, %Y') + " - " + datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['end']/1000),'%d %b, %Y')
             except:
                 pass
             subOrder.offerDiscount = s['offerDiscount']
             subOrder.unitPrice = s['offerPrice']
             merchantSubOrders.append(subOrder)
     def scrapeStoreOrders(self,):
         #collectionMap = {'palcedOn':1}
         orders = self._getActiveOrders()
         for order in orders:
+            order = obj(order)
-            print "Order", self.store_name, order['orderId']
+            print "Order", self.store_name, order.orderId
             try:
-                url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
+                url = order.orderTrackingUrl
                 page = fetchResponseUsingProxy(url)
                 #page=page.decode("utf-8")
-                soup = BeautifulSoup(page)
                 try:
+                    pq = PyQuery(page)
+                    subOrderStatusMap={}
+                    for el in pq('div.cardLayout.pad-10.mb-10'):
+                        elpq = PyQuery(el)
+                        try:
+                            subOrderId = elpq("div.subOrderId").text().split(":")[1].strip()
+                            subOrderStatusMap[subOrderId] = elpq
+                        except:
+                            pass
+                    closedForReco = {}
+                    for suborder in order.subOrders:
+                        if suborder.closed:
+                            if suborder.merchantSubOrderId in subOrderStatusMap:
+                                del subOrderStatusMap[suborder.merchantSubOrderId]
+                                continue
+                        if subOrderStatusMap.has_key(suborder.merchantSubOrderId):
+                            elpq = subOrderStatusMap.get(suborder.merchantSubOrderId)
+                            del subOrderStatusMap[suborder.merchantSubOrderId]
+                            if elpq("#trackLink").attr("href"):
+                                suborder.trackingUrl = elpq("#trackLink").attr("href")
+                            if elpq('span.subOrdStatusText').text():
+                                suborder.estimatedDeliveryDate=elpq('span.subOrdStatusText').text().strip()
+                            suborder.detailedStatus = elpq("div.orderStatus span").text().strip()
+                            if suborder.detailedStatus == 'Closed. New Order Placed':
+                                closedForReco[suborder.merchantSubOrderId] = suborder
+                            suborder.status = self._getStatusFromDetailedStatus(suborder.detailedStatus)
+                    if len(closedForReco) ==  len(subOrderStatusMap) and len(closedForReco)>0:
+                        productCode = ''
+                        allProductsSame = True
+                        for subOrderId, subo in closedForReco.iteritems():
+                            if productCode == '':
+                                productCode = subo.productCode
+                                continue
+                            if subo.productCode != productCode:
+                                allProductsSame = False
+                                break
+                        subOrderStatusMap
+                        if allProductsSame:
+                            print "singlereco", order.orderId
+                            suborderNew = obj(todict(suborder))
+                            for key, elpq in subOrderStatusMap.iteritems():
+                                suborderNew.merchantSubOrderId = key
+                                if elpq("#trackLink").attr("href"):
+                                    suborderNew.trackingUrl = elpq("#trackLink").attr("href")
+                                if elpq('span.subOrdStatusText').text():
+                                    suborderNew.estimatedDeliveryDate=elpq('span.subOrdStatusText').text().strip()
+                                suborderNew.detailedStatus = elpq("div.orderStatus span").text().strip()
+                                suborderNew.status = self._getStatusFromDetailedStatus(suborder.detailedStatus)
+                                order.subOrders.append(suborderNew)
+                        else:
+                            print "All products not same", order.orderId
+                    else:
+                        pass
+                        #Lookout for ordersummary page for exact mapping
-                    self.tryBParsing(order, soup)
+                    self.populateDerivedFields(order)
+                    self._updateToOrder(todict(order))
                 except:
                     traceback.print_exc()
+                    soup = BeautifulSoup(page)
+                    try:
+                        self.tryBParsing(order, soup)
+                    except:
+                        traceback.print_exc()
-                    sections = soup.findAll("section")
+                        sections = soup.findAll("section")
-                    orderEl = sections[1]
+                        orderEl = sections[1]
-                    orderTrs = orderEl.findAll("tr")
+                        orderTrs = orderEl.findAll("tr")
-                    placedOn = str(orderTrs[0].findAll("td")[1].text)
+                        placedOn = str(orderTrs[0].findAll("td")[1].text)
-                    sections.pop(0)
+                        sections.pop(0)
-                    sections.pop(0)
+                        sections.pop(0)
-                    subOrders = sections
+                        subOrders = sections
-                    bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
+                        bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
-                    closed = True
+                        closed = True
-                    for subOrderElement in subOrders:
+                        for subOrderElement in subOrders:
-                        div1 = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
+                            div1 = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
-                        if len(div1)<=0:
+                            if len(div1)<=0:
-                            raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
+                                raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
-                        subOrder = None
+                            subOrder = None
-                        breakFlag = False
+                            breakFlag = False
-                        for strDiv in str(div1).split("<div class=\"seperator\"></div>"):
+                            for strDiv in str(div1).split("<div class=\"seperator\"></div>"):
-                            div = BeautifulSoup(strDiv)
+                                div = BeautifulSoup(strDiv)
-                            divStr = str(div)
+                                divStr = str(div)
-                            divStr = divStr.replace("\n","").replace("\t", "")
+                                divStr = divStr.replace("\n","").replace("\t", "")
-                            updateMap = {}
+                                updateMap = {}
-                            for line in divStr.split("<br />"):
+                                for line in divStr.split("<br />"):
-                                if "Suborder ID" in line:
+                                    if "Suborder ID" in line:
-                                    merchantSubOrderId = re.findall(r'\d+', line)[0]
+                                        merchantSubOrderId = re.findall(r'\d+', line)[0]
-                                    #break if suborder is inactive
+                                        #break if suborder is inactive
-                                    subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
+                                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
-                                    if subOrder is None:
+                                        if subOrder is None:
-                                        subOrders = self.parseSubOrder(subOrderElement, placedOn)
+                                            subOrders = self.parseSubOrder(subOrderElement, placedOn)
-                                        self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict(subOrders)}}})
+                                            self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict(subOrders)}}})
-                                        print "Added new suborders to Order id - ", order['orderId']
+                                            print "Added new suborders to Order id - ", order['orderId']
-                                        closed = False
+                                            closed = False
-                                        breakFlag = True
+                                            breakFlag = True
-                                        break
+                                            break
-                                    elif subOrder['closed']:
+                                        elif subOrder['closed']:
-                                        breakFlag = True
+                                            breakFlag = True
-                                        break
+                                            break
-                                    else:
+                                        else:
-                                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
+                                            findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
-                                elif "Status :" in line:
+                                    elif "Status :" in line:
-                                    detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
+                                        detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
-                                    updateMap["subOrders.$.detailedStatus"] = detailedStatus
+                                        updateMap["subOrders.$.detailedStatus"] = detailedStatus
-                                    status = self._getStatusFromDetailedStatus(detailedStatus)
+                                        status = self._getStatusFromDetailedStatus(detailedStatus)
-                                    closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
+                                        closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
-                                    if status is not None:
+                                        if status is not None:
-                                        updateMap["subOrders.$.status"] = status
+                                            updateMap["subOrders.$.status"] = status
-                                    if detailedStatus == 'Closed For Vendor Reallocation':
+                                        if detailedStatus == 'Closed For Vendor Reallocation':
-                                        #if it is more than 6hours mark closed.
+                                            #if it is more than 6hours mark closed.
-                                        closeAt = subOrder.get("closeAt")
+                                            closeAt = subOrder.get("closeAt")
-                                        if closeAt is None:
+                                            if closeAt is None:
-                                            closeAt = datetime.now() + timedelta(hours=6)
+                                                closeAt = datetime.now() + timedelta(hours=6)
-                                            updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")
+                                                updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")
+                                            else:
+                                                closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")
+                                                if datetime.now() > closeAt:
+                                                    closedStatus = True
+                                        #Close if not applicable suborders are not closed
+                                        if utils.fromTimeStamp(order['createdOnInt'] + 35*86400*1000) < datetime.now() and subOrder['cashBackStatus']==utils.CB_NA:
+                                            closedStatus=True
+                                        if closedStatus:
+                                            #if status is closed then change the paybackStatus accordingly
+                                            updateMap["subOrders.$.closed"] = True
+                                            if status == Store.ORDER_DELIVERED:
+                                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:
+                                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
+                                            elif status == Store.ORDER_CANCELLED:
+                                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:
+                                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
                                         else:
-                                            closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")
-                                            if datetime.now() > closeAt:
-                                                closedStatus = True
-                                    #Close if not applicable suborders are not closed
-                                    if utils.fromTimeStamp(order['createdOnInt'] + 35*86400*1000) < datetime.now() and subOrder['cashBackStatus']==utils.CB_NA:
-                                        closedStatus=True
-                                    if closedStatus:
-                                        #if status is closed then change the paybackStatus accordingly
-                                        updateMap["subOrders.$.closed"] = True
-                                        if status == Store.ORDER_DELIVERED:
-                                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
-                                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
-                                        elif status == Store.ORDER_CANCELLED:
-                                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
-                                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
-                                    else:
-                                        closed = False
+                                            closed = False
-                                elif "Est. Shipping Date" in line:
+                                    elif "Est. Shipping Date" in line:
-                                    estimatedShippingDate = line.split(":")[1].strip()
+                                        estimatedShippingDate = line.split(":")[1].strip()
-                                    updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
+                                        updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
-                                elif "Est. Delivery Date" in line:
+                                    elif "Est. Delivery Date" in line:
-                                    estimatedDeliveryDate = line.split(":")[1].strip()
+                                        estimatedDeliveryDate = line.split(":")[1].strip()
-                                    updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
+                                        updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
-                                elif "Courier Name" in line:
+                                    elif "Courier Name" in line:
-                                    courierName = line.split(":")[1].strip()
+                                        courierName = line.split(":")[1].strip()
-                                    updateMap["subOrders.$.courierName"] = courierName
+                                        updateMap["subOrders.$.courierName"] = courierName
-                                elif "Tracking No" in line:
+                                    elif "Tracking No" in line:
-                                    trackingNumber = line.split(":")[1].strip()
+                                        trackingNumber = line.split(":")[1].strip()
-                                    updateMap["subOrders.$.trackingNumber"] = trackingNumber
+                                        updateMap["subOrders.$.trackingNumber"] = trackingNumber
-                            if breakFlag:
+                                if breakFlag:
-                                continue
+                                    continue
-                            bulk.find(findMap).update({'$set' : updateMap})
+                                bulk.find(findMap).update({'$set' : updateMap})
-                        bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
+                            bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
-                    result = bulk.execute()
+                        result = bulk.execute()
-                    tprint(result)
+                        tprint(result)
             except:
-                tprint("Could not update " + str(order['orderId']) + "For store " + self.getName())
-                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
                 traceback.print_exc()
+                tprint("Could not update " + str(order.orderId) + "For store " + self.getName())
+                self.db.merchantOrder.update({"orderId":order.orderId}, {"$set":{"parseError":True}})
     def tryBParsing(self, order, soup):
         orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)
         orderDetailDiv = orderDetailContainerDivs.pop(0)
         placedOn = orderDetailDiv.span.text.split(':')[1].strip()
 def main():
     #print todict([1,2,"3"])
     store = getStore(3)
+    #store.parseOrderRawHtml(332221, "3232311", 2, readSSh("/home/amit/snapdeal.html"), "https://m.snapdeal.com/purchaseMobileComplete?code=b92753bd7236bb3efbd6e8a0df46b962&order=9627657388")
-    #store.scrapeStoreOrders()
+    store.scrapeStoreOrders()
     #https://m.snapdeal.com/purchaseMobileComplete?code=3fbc8a02a1c4d3c4e906f46886de0464&order=5808451506
     #https://m.snapdeal.com/purchaseMobileComplete?code=9f4dfa49ff08a16d04c5e4bf519506fc&order=9611672826
-    orders = session.query(OrdersRaw).filter_by(store_id=3).filter_by(status='ORDER_NOT_CREATED').all()
+#    orders = list(session.query(OrdersRaw).filter_by(store_id=3).filter_by(status='ORDER_NOT_CREATED').all())
-    for o in orders:
+#    for o in orders:
-        result = store.parseOrderRawHtml(o.id, o.sub_tag, o.user_id, o.rawhtml, o.order_url)['result']
+#        result = store.parseOrderRawHtml(o.id, o.sub_tag, o.user_id, o.rawhtml, o.order_url)['result']
-        o.status = result
+#        o.status = result
-        session.commit()
+#    session.commit()
-        #store.parseOrderRawHtml(332221, "3232311", 2, readSSh("/home/amit/snapdeal.html"), "https://m.snapdeal.com/purchaseMobileComplete?code=3fbc8a02a1c4d3c4e906f46886de0464&order=5808451506")
+#    session.close()
     #store.scrapeStoreOrders()
     #store._isSubOrderActive(8, "5970688907")
     #store.scrapeAffiliate(datetime(2015,4,1))
     #store.scrapeStoreOrders()
     #store.parseInfo()
+class obj(object):
+    def __init__(self, d):
+        for a, b in d.items():
+            if isinstance(b, (list, tuple)):
+               setattr(self, a, [obj(x) if isinstance(x, dict) else x for x in b])
+            else:
+               setattr(self, a, obj(b) if isinstance(b, dict) else b)
 def todict(obj, classkey=None):
     if isinstance(obj, dict):
         data = {}
         for (k, v) in obj.items():

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/snapdeal.py – Rev 17389 → 17394