WebSVN – SmartDukaan – //trunk/PyDTR/src/dtr/sources/flipkart.py

'''
Created on Jan 15, 2015

@author: amit
'''
from dtr import main
import importlib
from dtr.main import getBrowserObject, getStore, ParseException
from dtr.main import sourceMap
from pymongo.mongo_client import MongoClient
from BeautifulSoup import BeautifulSoup
import re
from dtr.dao import AffiliateInfo, Order, SubOrder
import urllib

from pprint import pprint
USERNAME='saholic1@gmail.com'
PASSWORD='spice@2020'
ORDER_TRACK_URL='https://m.flipkart.com/order_details'
AFFILIATE_URL='https://www.flipkart.com/affiliate/login'


class Store(main.Store):
    OrderStatusMap = {
                      main.Store.ORDER_PLACED : ['In Progress','N/A'],
                      main.Store.ORDER_DELIVERED : ['Delivered'],
                      main.Store.ORDER_SHIPPED : ['In Transit'],
                      main.Store.ORDER_CANCELLED : ['Closed For Vendor Reallocation']
                      
                      }
    def __init__(self,store_id):
        client = MongoClient('mongodb://localhost:27017/')
        self.db = client.dtr
        super(Store, self).__init__(store_id)

    def getName(self):
        return "flipkart"
    
    def scrapeAffiliate(self, startDate=None, endDate=None):
        br = getBrowserObject()
        br.open(AFFILIATE_URL)
        print br.select_form(name="fk-signin")
        
    
    
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        br = getBrowserObject()
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
       
        response = br.open(url)
        page = ungzipResponse(response, br)
       
        
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
        
        sections = soup.findAll("div", {"class":"ui-app-card-body"})
        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})
        sections.pop(1)
        
        #fetching summary of the products
        for data in sections:
            name = data.findAll("span")
            i=0
            print "Summary Details of Flipkart"
            while i< len(name):
                if "key" in str(name[i]):
                    if "Grand Total" in str(name[i]):
                        #Total Amount
                        print "Total Amount " + name[i+1].text
                    elif "Status" in str(name[i]):
                        print "Summary Status " + name[i+1].text
                    elif "Order ID" in str(name[i]):
                        j=i+1
                        while j < i+5:
                            if "value" in str(name[j]):
                                print "Order ID " + str(j-i) + " " +name[j].text
                                j=j+1
                            else: 
                                break        
                    elif "Order Date" in str(name[i]):
                        print "Order Date " + name[i+1].text    
                i=i+1    
        print "Details of Sub Orders"
        #fetching suborders details
        for subOrder in mainOrder:
            subOrd = subOrder.findAll("li")
           
            subOrd.pop(len(subOrd)-1)
            for productInfo in subOrd:
                y = productInfo.findAll("span")
                for suborderId in y:
                    if "value emp" in str(suborderId):
                        #Sub Order ID
                        print "Sub Order ID"  + suborderId.text
                g = productInfo.findAll("div",{"class":"product-image-block"})
                for imageUrl in g:
                    print str(imageUrl.find("img")['src'])         
                x = productInfo.findAll("div",{"class":"product-info"})
                for maik in x:
                    #Product title
                    print maik.find("a",{"class":"product-title"}).text
                    print str(maik.find("a")['href'])
                    start='pid='
                    s=str(maik.find("a")['href'])
                    print re.findall(re.escape(start)+"(.*)",s)[0]
                    mname = maik.findAll("span")
                    k=0
                    while k<len(mname):
                        if "note" in str(mname[k]):
                            if "Color:" in str(mname[k]):
                                #Color
                                print "Color " + mname[k+1].text
                            elif "Qty:" in str(mname[k]):
                                #Quantity of sub item
                                print "Quantity "+mname[k+1].text
                            elif "Subtotal:" in str(mname[k]):
                                #SubTotal for the item
                                print "Sub Total"+mname[k+1].text
                            elif "Delivery:" in str(mname[k]):
                                #Delivery Date for sub order
                                print "Delivery Date " +mname[k+1].text
                        if "key text-dim" in str(mname[k]):
                            if "Seller" in str(mname[k]):
                                print "Seller  URL : http://m.flipkart.com"+maik.findAll("a")[1]['href']
                                print "Seller "+maik.findAll("a")[1].text                
                        k=k+1           
                status=-1    
                #To track shipping details         
                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})
                for trackingStatus in n:
                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                    if "approveDetails-complete" in str(tr):
                        if "processingDetails-complete" in str(tr):
                            if "shippingDetails-complete" in str(tr):
                                if "delivery-complete" in str(tr):
                                    status = 3
                                else:     
                                    status = 2
                            else:
                                status = 1 
                        else: 
                            status = 0
                    elif "approveDetails-ongoing" in str(tr):
                        status=0             
                    if "dead" in str(tr):
                        status = 4    
                print "Sub Order Status " + str(status)                      
                f = productInfo.findAll("div",{"class":"c-tabs-content m-top active"})
                for trackingDetailsActive in f:
                    print "Sub order tracking description " + trackingDetailsActive.text
                
    
    def flipkartOrderTracking(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        br = getBrowserObject()
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
       
        response = br.open(url)
        page = ungzipResponse(response, br)
       
        
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
        
        
        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})
        #fetching suborders details
        for subOrder in mainOrder:
            subOrd = subOrder.findAll("li")
           
            subOrd.pop(len(subOrd)-1)
            for productInfo in subOrd:
                status=-1    
                #To track shipping details         
                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})
                for trackingStatus in n:
                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                    if "approveDetails-complete" in str(tr):
                        print "Here"
                        if "processingDetails-complete" in str(tr):
                            print "There"
                            if "shippingDetails-complete" in str(tr):
                                print "Share"
                                if "delivery-complete" in str(tr):
                                    print "Last"
                                    status = 3
                                else:     
                                    status = 2
                            else:
                                status = 1 
                        else: 
                            status = 0     
                    if "dead" in str(tr):
                        status = 4    
                print status                      
               
     
                
def _saveToOrderFlipkart(self, order):
        collection = self.db.merchantOrder
        order = collection.insert(order)

def main():
  
    store = getStore(2)
    store.scrapeAffiliate()
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')

def ungzipResponse(r,b):
        headers = r.info()
        if headers['Content-Encoding']=='gzip':
            import gzip
        print "********************"
        print "Deflating gzip response"
        print "********************"
        gz = gzip.GzipFile(fileobj=r, mode='rb')
        html = gz.read()
        gz.close()
        return html    
    
if __name__ == '__main__':
        main()
        
def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj
Subversion Repositories SmartDukaan

(root)//trunk/PyDTR/src/dtr/sources/flipkart.py – Rev 13679
