WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/dtr/sources/flipkart.py

 '''
 Created on Jan 15, 2015
 @author: amit
 '''
+from BeautifulSoup import BeautifulSoup
 from dtr import main
-import importlib
+from dtr.dao import AffiliateInfo, Order, SubOrder
-from dtr.main import getBrowserObject, getStore, ParseException
+from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
-from dtr.main import sourceMap
+    Store as MStore, sourceMap
+from pprint import pprint
 from pymongo.mongo_client import MongoClient
+import hashlib
-from BeautifulSoup import BeautifulSoup
+import importlib
+import json
+import mechanize
 import re
-from dtr.dao import AffiliateInfo, Order, SubOrder
 import urllib
-from pprint import pprint
 USERNAME='saholic1@gmail.com'
 PASSWORD='spice@2020'
 ORDER_TRACK_URL='https://m.flipkart.com/order_details'
 AFFILIATE_URL='https://www.flipkart.com/affiliate/login'
+AFFILIATE_LOGIN_URL='https://www.flipkart.com/affiliate/a_login'
+AFF_REPORT_URL='http://www.flipkart.com/affiliate/reports/ordersReport?order_status_filter=%s&startdate=%s&enddate=%s'
+AFF_STATUS_CANCELLED='cancelled'
+AFF_STATUS_APPROVED='approved'
+AFF_STATUS_DISAPPROVED='disapproved'
+AFF_STATUS_PENDING='pending'
 class Store(main.Store):
     OrderStatusMap = {
-                      main.Store.ORDER_PLACED : ['In Progress','N/A'],
+                      main.Store.ORDER_PLACED : ['Approval', 'Processing'],
-                      main.Store.ORDER_DELIVERED : ['Delivered'],
+                      main.Store.ORDER_DELIVERED : ['Shipping'],
                       main.Store.ORDER_SHIPPED : ['In Transit'],
-                      main.Store.ORDER_CANCELLED : ['Closed For Vendor Reallocation']
+                      main.Store.ORDER_CANCELLED : ['']
+                      }
     def __init__(self,store_id):
         client = MongoClient('mongodb://localhost:27017/')
         self.db = client.dtr
         super(Store, self).__init__(store_id)
     def getName(self):
         return "flipkart"
+    def scrapeStoreOrders(self,):
+        orders = self._getActiveOrders()
     def scrapeAffiliate(self, startDate=None, endDate=None):
         br = getBrowserObject()
         br.open(AFFILIATE_URL)
-        print br.select_form(name="fk-signin")
+        response = br.response()  # copy
+        token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
+        data = {'__FK':token,
+                'email':'saholic1@gmail.com',
+                'password':'e8aacf6fc1e3998186a4a8e56e428f66'
+        }
+        br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
+        for status in [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_PENDING, AFF_STATUS_DISAPPROVED]:
+            br.open(AFF_REPORT_URL % (status, '2015-01-02', '2015-02-01'))
+            page = ungzipResponse(br.response())
+            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
+            try:
+                tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]
+            except:
+                continue
+            trElements = tableElement.findAll('tr')
+            trElements.pop(0)
+            for trElement in trElements:
+                tdElements = trElement.findAll('td')
+                productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]
+                quantity =int(tdElements[3].text)
+                price = int(float(tdElements[2].text))
+                payOut = tdElements[6].text
+                subTagId = tdElements[7].text
+                reason = tdElements[9].text
     def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
         br = getBrowserObject()
         url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
         response = br.open(url)
-        page = ungzipResponse(response, br)
+        page = ungzipResponse(response)
+        merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]
+        print merchantOrderId
         soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
         sections = soup.findAll("div", {"class":"ui-app-card-body"})
-        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})
         sections.pop(1)
-        #fetching summary of the products
+        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
         for data in sections:
             name = data.findAll("span")
             i=0
-            print "Summary Details of Flipkart"
             while i< len(name):
                 if "key" in str(name[i]):
                     if "Grand Total" in str(name[i]):
                         #Total Amount
-                        print "Total Amount " + name[i+1].text
+                        merchantOrder.paidAmount = re.findall(r'\d+', name[i+1].text)[0]
-                    elif "Status" in str(name[i]):
-                        print "Summary Status " + name[i+1].text
-                    elif "Order ID" in str(name[i]):
-                        j=i+1
-                        while j < i+5:
-                            if "value" in str(name[j]):
-                                print "Order ID " + str(j-i) + " " +name[j].text
-                                j=j+1
-                            else:
-                                break
                     elif "Order Date" in str(name[i]):
-                        print "Order Date " + name[i+1].text
+                        merchantOrder.placedOn = name[i+1].text
                 i=i+1
+        merchantOrder.merchantOrderId = merchantOrderId
+        mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
+        fkSubOrders = mainOrder.findAll("li")
+        #remove unwanted list
+        fkSubOrders.pop(-1)
+        subOrders = []
-        print "Details of Sub Orders"
+        merchantOrder.subOrders = subOrders
         #fetching suborders details
-        for subOrder in mainOrder:
+        for subOrder in fkSubOrders:
-            subOrd = subOrder.findAll("li")
+            y = subOrder.find("header").findAll("span")
+            for suborderId in y:
-            subOrd.pop(len(subOrd)-1)
+                if "value emp" in str(suborderId):
+                    merchantSubOrderId = suborderId.text
-            for productInfo in subOrd:
+                    break
-                y = productInfo.findAll("span")
+            orderItems = subOrder.find("ul").findAll("div", recursive=False)
-                for suborderId in y:
+            i=0
-                    if "value emp" in str(suborderId):
+            for orderItem in orderItems:
-                        #Sub Order ID
+                merchantOrder.placedOn
-                        print "Sub Order ID"  + suborderId.text
+                merchantSubOrderId = merchantSubOrderId + str(i)
-                g = productInfo.findAll("div",{"class":"product-image-block"})
+                divs = orderItem.findAll('div', recursive=False)
-                for imageUrl in g:
+                content = divs[0]
-                    print str(imageUrl.find("img")['src'])
+                orderTracking = divs[2]
-                x = productInfo.findAll("div",{"class":"product-info"})
+                orderTrackingDet = divs[3]
-                for maik in x:
-                    #Product title
+                imgUrl = content.find('img')['src']
+                lineDet = content.find("div",{"class":"product-info"})
-                    print maik.find("a",{"class":"product-title"}).text
+                productTitle = lineDet.find("a",{"class":"product-title"}).text
-                    print str(maik.find("a")['href'])
+                productUrl =  str(lineDet.find("a")['href'])
-                    start='pid='
+                start='pid='
-                    s=str(maik.find("a")['href'])
+                s=str(lineDet.find("a")['href'])
-                    print re.findall(re.escape(start)+"(.*)",s)[0]
+                productCode = re.findall(re.escape(start)+"(.*)",s)[0]
-                    mname = maik.findAll("span")
+                mname = lineDet.findAll("span")
-                    k=0
+                k=0
-                    while k<len(mname):
+                while k<len(mname):
-                        if "note" in str(mname[k]):
+                    if "note" in str(mname[k]):
-                            if "Color:" in str(mname[k]):
+                        if "Color:" in str(mname[k]):
+                            productTitle = productTitle + " " + mname[k+1].text
-                                #Color
-                                print "Color " + mname[k+1].text
+                        elif "Qty:" in str(mname[k]):
-                            elif "Qty:" in str(mname[k]):
+                            quantity = int(mname[k+1].text)
-                                #Quantity of sub item
+                        elif "Subtotal:" in str(mname[k]):
-                                print "Quantity "+mname[k+1].text
+                            amountPaid = re.findall(r'\d+', mname[k+1].text)[0]
-                            elif "Subtotal:" in str(mname[k]):
+                        elif "Delivery:" in str(mname[k]):
-                                #SubTotal for the item
+                            #Delivery Date for sub order
-                                print "Sub Total"+mname[k+1].text
+                            print "Delivery Date " +mname[k+1].text
+                    k=k+1
+                merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)
-                            elif "Delivery:" in str(mname[k]):
+                merchantsubOrder.imgUrl  = imgUrl
-                                #Delivery Date for sub order
+                merchantsubOrder.productCode = productCode
-                                print "Delivery Date " +mname[k+1].text
+                cashbackAmount = self.getCashbackAmount(subOrder.productCode, amountPaid)
-                        if "key text-dim" in str(mname[k]):
+                cashbackStatus = Store.CB_INIT
+                if cashbackAmount <= 0:
-                            if "Seller" in str(mname[k]):
+                    cashbackStatus = Store.CB_NA
-                                print "Seller  URL : http://m.flipkart.com"+maik.findAll("a")[1]['href']
+                merchantsubOrder.cashBackAmount = cashbackAmount
-                                print "Seller "+maik.findAll("a")[1].text
+                merchantsubOrder.cashBackStatus = cashbackStatus
-                        k=k+1
-                status=-1
+                subOrders.append(merchantsubOrder)
                 #To track shipping details
-                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})
-                for trackingStatus in n:
+                status=-1
-                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
+                tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
-                    if "approveDetails-complete" in str(tr):
+                if "approveDetails-complete" in str(tr):
-                        if "processingDetails-complete" in str(tr):
+                    if "processingDetails-complete" in str(tr):
-                            if "shippingDetails-complete" in str(tr):
+                        if "shippingDetails-complete" in str(tr):
-                                if "delivery-complete" in str(tr):
+                            if "delivery-complete" in str(tr):
-                                    status = 3
+                                status = MStore.ORDER_DELIVERED
-                                else:
+                            else:
-                                    status = 2
+                                status = MStore.ORDER_SHIPPED
-                            else:
+                        else:
-                                status = 1
+                            status = MStore.ORDER_PLACED
-                        else:
+                    else:
-                            status = 0
+                        status = MStore.ORDER_PLACED
-                    elif "approveDetails-ongoing" in str(tr):
+                elif str(tr) in ["approveDetails-ongoing"]:
-                        status=0
+                    status=MStore.ORDER_PLACED
-                    if "dead" in str(tr):
+                if "dead" in str(tr):
-                        status = 4
+                    status = MStore.ORDER_CANCELLED
                 print "Sub Order Status " + str(status)
-                f = productInfo.findAll("div",{"class":"c-tabs-content m-top active"})
-                for trackingDetailsActive in f:
-                    print "Sub order tracking description " + trackingDetailsActive.text
+                trackingDetailsActive = orderTrackingDet.find("div",{"class":"c-tabs-content m-top active"})
+                print "Sub order tracking description " + trackingDetailsActive.text
+        self._saveToOrder(todict(merchantOrder))
     def flipkartOrderTracking(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
         br = getBrowserObject()
         url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
 def _saveToOrderFlipkart(self, order):
         collection = self.db.merchantOrder
         order = collection.insert(order)
+def hex_md5(password):
+    m = hashlib.md5()
+    print(m.digest())
 def main():
     store = getStore(2)
-    store.scrapeAffiliate()
+    #store.scrapeAffiliate()
+    store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3019717576543723&token=14abca87835b66da4ce8fa9d7594d95f&src=or&pr=1')
     #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
     #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
     #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
     #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
     #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
     #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')
+    #hex_md5('spice@2020')
-def ungzipResponse(r,b):
-        headers = r.info()
-        if headers['Content-Encoding']=='gzip':
-            import gzip
-        print "********************"
-        print "Deflating gzip response"
-        print "********************"
-        gz = gzip.GzipFile(fileobj=r, mode='rb')
-        html = gz.read()
-        gz.close()
-        return html
 if __name__ == '__main__':
         main()
 def todict(obj, classkey=None):
     if isinstance(obj, dict):
         data = {}
         for (k, v) in obj.items():
             data[k] = todict(v, classkey)
             if not callable(value) and not key.startswith('_')])
         if classkey is not None and hasattr(obj, "__class__"):
             data[classkey] = obj.__class__.__name__
         return data
     else:
-        return obj
         return obj

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/flipkart.py – Rev 13679 → 13690