WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/dtr/sources/snapdeal.py

 '''
 Created on Jan 15, 2015
 @author: amit
 '''
-from BeautifulSoup import BeautifulSoup
+from bs4 import BeautifulSoup
 from bson.binary import Binary
 from datetime import datetime, date, timedelta
 from dtr import main
 from dtr.dao import AffiliateInfo, Order, SubOrder
 from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
 import pymongo
 import re
 import time
 import traceback
 import urllib
+from urlparse import urlparse, parse_qs
 USERNAME='profittill2@gmail.com'
 PASSWORD='spice@2020'
 AFFILIATE_URL='http://affiliate.snapdeal.com'
 POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
     def _getLastSaleDate(self,):
         lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})
         if lastDaySaleObj is None:
             return datetime.min
+    def _parseB(self, orderId, subTagId, userId, page, orderSuccessUrl):
+        soup = BeautifulSoup(page)
+        orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)
+        orderDetailDiv = orderDetailContainerDivs.pop(0)
+        paymentDetailDiv  = orderDetailContainerDivs.pop(0)
+        subOrders = orderDetailContainerDivs
+        placedOn = orderDetailDiv.span.text.split(':')[1].strip()
+        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
+        merchantOrder.placedOn = placedOn
+        merchantOrder.merchantOrderId = parse_qs(urlparse(orderSuccessUrl).query)['order'][0]
+        paymentDivs = paymentDetailDiv.findAll('div', recursive=False)
+        paymentDivs.pop(0)
+        for orderTr in paymentDivs:
+            orderTrString = str(orderTr)
+            if "Total Amount Paid" in orderTrString:
+                amountPaid = orderTr.div.find('div', {'class':'detailBlock'}).text.strip()
+                merchantOrder.paidAmount = int(re.findall(r'\d+', amountPaid)[0])
+            elif "Total Amount" in orderTrString:
+                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]
+            elif "Delivery Charges" in orderTrString:
+                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]
+            elif "Discount Applied" in orderTrString:
+                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]
+            elif "Offer Discount" in orderTrString:
+                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]
+        merchantSubOrders = []
+        for subOrderElement in subOrders:
+            subOrder = self.parseSubOrderB(subOrderElement, placedOn)
+            merchantSubOrders.append(subOrder)
+        merchantOrder.subOrders = merchantSubOrders
+        return merchantOrder
     def _parse(self, orderId, subTagId, userId, page, orderSuccessUrl):
         #page=page.decode("utf-8")
-        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
+        soup = BeautifulSoup(page)
         #orderHead = soup.find(name, attrs, recursive, text)
         sections = soup.findAll("section")
         #print sections
                 elif "Tracking No" in line:
                     subOrder.trackingNumber = line.split(":")[1].strip()
             subOrders.append(subOrder)
         return subOrders
+    def parseSubOrderB(self, subOrderElement, placedOn):
+        subOrders = []
+        prodDivs = subOrderElement.findAll('div', recursive=False)
+        prodDetailDiv = prodDivs[1].findAll('div', recursive=False)
+        offerDiscount = 0
+        deliveryCharges = None
+        amountPaid = 0
+        sdCash = 0
+        unitPrice = 0
+        paymentDivs = prodDivs[2].findAll('div', recursive=False)
+        for paymentDiv in paymentDivs:
+            strPaymentDiv = str(paymentDiv)
+            if "Unit Price" in strPaymentDiv:
+                unitPrice = int(re.findall(r'\d+', strPaymentDiv)[0])
+            if "Offer Discount" in strPaymentDiv:
+                offerDiscount = int(re.findall(r'\d+', strPaymentDiv)[0])
+            elif "SD Cash:" in strPaymentDiv:
+                sdCash =   int(re.findall(r'\d+', strPaymentDiv)[0])
+            elif "Delivery Charges" in strPaymentDiv:
+                deliveryCharges =   int(re.findall(r'\d+', strPaymentDiv)[0])
+            elif "Subtotal" in strPaymentDiv:
+                amountPaid =   int(re.findall(r'\d+', paymentDiv.find('div', {'class':'itemPriceDetail'}).text)[0])
+        amount = unitPrice - offerDiscount - sdCash
+        imgDiv = prodDetailDiv[0]
+        otherDiv = prodDetailDiv[1]
+        productTitle = otherDiv.find('div',{'class':'orderName'}).text.strip()
+        productUrl = imgDiv.a['href']
+        subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)
+        subOrder.detailedStatus = otherDiv.find('div',{'class':'orderStatus'}).span.text.strip()
+        deliveryStatus = otherDiv.find('div',{'class':'orderDelivery'})
+        if deliveryStatus is not None:
+            delString = deliveryStatus.text.strip()
+            arr = delString.split(':')
+            if "On" in arr[0]:
+                subOrder.deliveredOn = arr[1].strip()
+            elif "Exp. Delivery by" in arr[0]:
+                subOrder.estimatedDeliveryDate = arr[1].strip()
+            else:
+                subOrder.estimatedShippingDate = arr[1].strip()
+        subOrder.imgUrl = imgDiv.a.img['src']
+        subOrder.productCode = re.findall(r'\d+$', productUrl)[0]
+        subOrder.deliveryCharges = deliveryCharges
+        subOrder.offerDiscount = offerDiscount
+        subOrder.unitPrice = int(unitPrice)
+        cashbackStatus = Store.CB_NA
+        cashbackAmount = 0
+        percentage = 0
+        if amountPaid > 0:
+            (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)
+            if cashbackAmount > 0:
+                cashbackStatus = Store.CB_PENDING
+        subOrder.cashBackStatus = cashbackStatus
+        subOrder.cashBackAmount = cashbackAmount
+        subOrder.cashBackPercentage = percentage
+        courierDet = subOrderElement.find('div', {'class':'courierDetail'})
+        if courierDet is not None:
+            subOrder.courierName = courierDet.span.text.strip()
+        trackingDet = subOrderElement.find('div', {'class':'trackingNo'})
+        if trackingDet is not None:
+            subOrder.trackingUrl = trackingDet.span.a['href']
+            subOrder.trackingNumber = trackingDet.span.a.text.strip()
+        subOrders.append(subOrder)
+        return subOrder
     def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
                     #print merchantOrder
         resp = {}
         try:
             br = getBrowserObject()
             url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
             page = br.open(url)
             page = ungzipResponse(page)
+            try:
+                merchantOrder = self._parseB(orderId, subTagId, userId, page, orderSuccessUrl)
+            except:
-            merchantOrder = self._parse(orderId, subTagId, userId, page, orderSuccessUrl)
+                merchantOrder = self._parse(orderId, subTagId, userId, page, orderSuccessUrl)
             merchantOrder.orderTrackingUrl = url
             if self._saveToOrder(todict(merchantOrder)):
                 resp['result'] = 'ORDER_CREATED'
             else:
             except:
                 tprint("Could not update " + str(order['orderId']) + "For store " + self.getName())
                 self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
                 traceback.print_exc()
+    def tryBParsing(self, order, soup):
+        orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)
+        orderDetailDiv = orderDetailContainerDivs.pop(0)
+        placedOn = orderDetailDiv.span.text.split(':')[1].strip()
+        orderDetailContainerDivs.pop(0)
+        subOrders = orderDetailContainerDivs
+        bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
+        closed = True
+        for subOrderElement in subOrders:
+            prodDivs = subOrderElement.findAll('div', recursive=False)
+            merchantSubOrderId = prodDivs[0].text.split(':')[1].strip()
+            prodDetailDiv = prodDivs[1].findAll('div', recursive=False)
+            imgDiv = prodDetailDiv[0]
+            otherDiv = prodDetailDiv[1]
+            subOrder = None
+            breakFlag = False
+            div = None
+            divStr = str(div)
+            divStr = divStr.replace("\n","").replace("\t", "")
+            updateMap = {}
+            for line in divStr.split("<br />"):
+                if "Suborder ID" in line:
+                    merchantSubOrderId = re.findall(r'\d+', line)[0]
+                    #break if suborder is inactive
+                    subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
+                    if subOrder is None:
+                        subOrders = self.parseSubOrder(subOrderElement, placedOn)
+                        self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict(subOrders)}}})
+                        print "Added new suborders to Order id - " + order['orderId']
+                        closed = False
+                        breakFlag = True
+                        break
+                    elif subOrder['closed']:
+                        breakFlag = True
+                        break
+                    else:
+                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
+                elif "Status :" in line:
+                    detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
+                    updateMap["subOrders.$.detailedStatus"] = detailedStatus
+                    status = self._getStatusFromDetailedStatus(detailedStatus)
+                    closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
+                    updateMap["subOrders.$.status"] = status
+                    if detailedStatus == 'Closed For Vendor Reallocation':
+                        #if it is more than 6hours mark closed.
+                        closeAt = subOrder.get("closeAt")
+                        if closeAt is None:
+                            closeAt = datetime.now() + timedelta(hours=6)
+                            updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")
+                        else:
+                            closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")
+                            if datetime.now() > closeAt:
+                                closedStatus = True
+                    if closedStatus:
+                        #if status is closed then change the paybackStatus accordingly
+                        updateMap["subOrders.$.closed"] = True
+                        if status == Store.ORDER_DELIVERED:
+                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
+                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
+                        elif status == Store.ORDER_CANCELLED:
+                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
+                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
+                    else:
+                        closed = False
+                elif "Est. Shipping Date" in line:
+                    estimatedShippingDate = line.split(":")[1].strip()
+                    updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
+                elif "Est. Delivery Date" in line:
+                    estimatedDeliveryDate = line.split(":")[1].strip()
+                    updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
+                elif "Courier Name" in line:
+                    courierName = line.split(":")[1].strip()
+                    updateMap["subOrders.$.courierName"] = courierName
+                elif "Tracking No" in line:
+                    trackingNumber = line.split(":")[1].strip()
+                    updateMap["subOrders.$.trackingNumber"] = trackingNumber
+            if breakFlag:
+                continue
+            bulk.find(findMap).update({'$set' : updateMap})
+        bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
+        result = bulk.execute()
+        tprint(result)
     def _saveToAffiliate(self, offers):
         collection = self.db.snapdealOrderAffiliateInfo
         mcollection = self.db.merchantOrder
         for offer in offers:
     return urllib.urlencode(parameters)
 def main():
     #print todict([1,2,"3"])
     store = getStore(3)
-    print store.parseOrderRawHtml(3221, "32323", 2, "323243", "https://m.snapdeal.com/purchaseMobileComplete?code=8943993fa3678d480157e7ed4292ccd5&order=5810418949")
+    store.parseOrderRawHtml(33222, "32323", 2, "323243", "https://m.snapdeal.com/purchaseMobileComplete?code=561009a4a0c043e2f537f558c8531580&order=5822820967")
     #store.scrapeStoreOrders()
     #store._isSubOrderActive(8, "5970688907")
     #store.scrapeAffiliate()
     #store.scrapeStoreOrders()

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/snapdeal.py – Rev 14847 → 14854