WebSVN – SmartDukaan – /trunk/PyDTR/src/dtr/sources/flipkart.py

'''
Created on Jan 15, 2015

@author: amit
'''
from BeautifulSoup import BeautifulSoup
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
    Store as MStore, sourceMap
from pprint import pprint
from pymongo.mongo_client import MongoClient
import hashlib
import importlib
import json
import mechanize
import re
import urllib

USERNAME='saholic1@gmail.com'
PASSWORD='spice@2020'
ORDER_TRACK_URL='https://m.flipkart.com/order_details'
AFFILIATE_URL='https://www.flipkart.com/affiliate/login'
AFFILIATE_LOGIN_URL='https://www.flipkart.com/affiliate/a_login'
AFF_REPORT_URL='http://www.flipkart.com/affiliate/reports/ordersReport?order_status_filter=%s&startdate=%s&enddate=%s'
AFF_STATUS_CANCELLED='cancelled'
AFF_STATUS_APPROVED='approved'
AFF_STATUS_DISAPPROVED='disapproved'
AFF_STATUS_PENDING='pending'


class Store(main.Store):
    OrderStatusMap = {
                      main.Store.ORDER_PLACED : ['Approval', 'Processing'],
                      main.Store.ORDER_DELIVERED : ['Shipping'],
                      main.Store.ORDER_SHIPPED : ['In Transit'],
                      main.Store.ORDER_CANCELLED : ['']
                      
                      }
    def __init__(self,store_id):
        client = MongoClient('mongodb://localhost:27017/')
        self.db = client.dtr
        super(Store, self).__init__(store_id)

    def getName(self):
        return "flipkart"
    
    def scrapeStoreOrders(self,):
        orders = self._getActiveOrders()
    
    def scrapeAffiliate(self, startDate=None, endDate=None):
        br = getBrowserObject()
        br.open(AFFILIATE_URL)
        response = br.response()  # copy
        token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
        data = {'__FK':token,
                'email':'saholic1@gmail.com',
                'password':'e8aacf6fc1e3998186a4a8e56e428f66'
        }
        br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
        for status in [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_PENDING, AFF_STATUS_DISAPPROVED]:
            br.open(AFF_REPORT_URL % (status, '2015-01-02', '2015-02-01'))
            page = ungzipResponse(br.response())
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            try:
                tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]
            except:
                continue
            trElements = tableElement.findAll('tr')
            trElements.pop(0)
            for trElement in trElements:
                tdElements = trElement.findAll('td')
                productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]
                quantity =int(tdElements[3].text)
                price = int(float(tdElements[2].text))
                payOut = tdElements[6].text
                subTagId = tdElements[7].text
                reason = tdElements[9].text
    
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        br = getBrowserObject()
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
       
        response = br.open(url)
        page = ungzipResponse(response)
        
        merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]
        print merchantOrderId
        
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
        
        sections = soup.findAll("div", {"class":"ui-app-card-body"})
        sections.pop(1)

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
        for data in sections:
            name = data.findAll("span")
            i=0
            while i< len(name):
                if "key" in str(name[i]):
                    if "Grand Total" in str(name[i]):
                        #Total Amount
                        merchantOrder.paidAmount = re.findall(r'\d+', name[i+1].text)[0]
                    elif "Order Date" in str(name[i]):
                        merchantOrder.placedOn = name[i+1].text 
                i=i+1    
        
        merchantOrder.merchantOrderId = merchantOrderId

        mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
        fkSubOrders = mainOrder.findAll("li")
        #remove unwanted list
        fkSubOrders.pop(-1)
        subOrders = []
        merchantOrder.subOrders = subOrders

        #fetching suborders details
        for subOrder in fkSubOrders:
            y = subOrder.find("header").findAll("span")
            for suborderId in y:
                if "value emp" in str(suborderId):
                    merchantSubOrderId = suborderId.text
                    break
            orderItems = subOrder.find("ul").findAll("div", recursive=False)
            i=0
            for orderItem in orderItems:
                merchantOrder.placedOn
                merchantSubOrderId = merchantSubOrderId + str(i)
                divs = orderItem.findAll('div', recursive=False)
                content = divs[0]
                orderTracking = divs[2]
                orderTrackingDet = divs[3]
                
                
                imgUrl = content.find('img')['src']
                lineDet = content.find("div",{"class":"product-info"})
                productTitle = lineDet.find("a",{"class":"product-title"}).text
                productUrl =  str(lineDet.find("a")['href'])

                start='pid='
                s=str(lineDet.find("a")['href'])
                productCode = re.findall(re.escape(start)+"(.*)",s)[0]
                mname = lineDet.findAll("span")
                k=0
                while k<len(mname):
                    if "note" in str(mname[k]):
                        if "Color:" in str(mname[k]):
                            productTitle = productTitle + " " + mname[k+1].text
                            
                        elif "Qty:" in str(mname[k]):
                            quantity = int(mname[k+1].text)
                        elif "Subtotal:" in str(mname[k]):
                            amountPaid = re.findall(r'\d+', mname[k+1].text)[0]
                        elif "Delivery:" in str(mname[k]):
                            #Delivery Date for sub order
                            print "Delivery Date " +mname[k+1].text
                    k=k+1
                merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)
                merchantsubOrder.imgUrl  = imgUrl
                merchantsubOrder.productCode = productCode
                
                cashbackAmount = self.getCashbackAmount(subOrder.productCode, amountPaid)
                cashbackStatus = Store.CB_INIT
                if cashbackAmount <= 0:
                    cashbackStatus = Store.CB_NA
                merchantsubOrder.cashBackAmount = cashbackAmount
                merchantsubOrder.cashBackStatus = cashbackStatus      
                
                subOrders.append(merchantsubOrder)
                #To track shipping details         
                status=-1    
                tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                if "approveDetails-complete" in str(tr):
                    if "processingDetails-complete" in str(tr):
                        if "shippingDetails-complete" in str(tr):
                            if "delivery-complete" in str(tr):
                                status = MStore.ORDER_DELIVERED
                            else:     
                                status = MStore.ORDER_SHIPPED
                        else:
                            status = MStore.ORDER_PLACED
                    else: 
                        status = MStore.ORDER_PLACED
                elif str(tr) in ["approveDetails-ongoing"]:
                    status=MStore.ORDER_PLACED             
                if "dead" in str(tr):
                    status = MStore.ORDER_CANCELLED    
                print "Sub Order Status " + str(status)                      
                
                trackingDetailsActive = orderTrackingDet.find("div",{"class":"c-tabs-content m-top active"})
                print "Sub order tracking description " + trackingDetailsActive.text
        self._saveToOrder(todict(merchantOrder))
    
    def flipkartOrderTracking(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        br = getBrowserObject()
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
       
        response = br.open(url)
        page = ungzipResponse(response, br)
       
        
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
        
        
        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})
        #fetching suborders details
        for subOrder in mainOrder:
            subOrd = subOrder.findAll("li")
           
            subOrd.pop(len(subOrd)-1)
            for productInfo in subOrd:
                status=-1    
                #To track shipping details         
                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})
                for trackingStatus in n:
                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                    if "approveDetails-complete" in str(tr):
                        print "Here"
                        if "processingDetails-complete" in str(tr):
                            print "There"
                            if "shippingDetails-complete" in str(tr):
                                print "Share"
                                if "delivery-complete" in str(tr):
                                    print "Last"
                                    status = 3
                                else:     
                                    status = 2
                            else:
                                status = 1 
                        else: 
                            status = 0     
                    if "dead" in str(tr):
                        status = 4    
                print status                      
               
     
                
def _saveToOrderFlipkart(self, order):
        collection = self.db.merchantOrder
        order = collection.insert(order)


    
def hex_md5(password):
    m = hashlib.md5()
    print(m.digest())

def main():
  
    store = getStore(2)
    #store.scrapeAffiliate()
    store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3019717576543723&token=14abca87835b66da4ce8fa9d7594d95f&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')
    #hex_md5('spice@2020')


if __name__ == '__main__':
        main()
        
        
    
        
def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj
Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/flipkart.py – Rev 13690
