Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder

from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \

    Store as MStore, sourceMap

from pprint import pprint

from pymongo.mongo_client import MongoClient

import hashlib

import importlib

import json

import mechanize

import re

import urllib

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

ORDER_TRACK_URL='https://m.flipkart.com/order_details'

AFFILIATE_URL='https://www.flipkart.com/affiliate/login'

AFFILIATE_LOGIN_URL='https://www.flipkart.com/affiliate/a_login'

AFF_REPORT_URL='http://www.flipkart.com/affiliate/reports/ordersReport?order_status_filter=%s&startdate=%s&enddate=%s'

AFF_STATUS_CANCELLED='cancelled'

AFF_STATUS_APPROVED='approved'

AFF_STATUS_DISAPPROVED='disapproved'

AFF_STATUS_PENDING='pending'

class Store(main.Store):

    OrderStatusMap = {

                      main.Store.ORDER_PLACED : ['Approval', 'Processing'],

                      main.Store.ORDER_DELIVERED : ['Shipping'],

                      main.Store.ORDER_SHIPPED : ['In Transit'],

                      main.Store.ORDER_CANCELLED : ['']

                      }

    def __init__(self,store_id):

        client = MongoClient('mongodb://localhost:27017/')

        self.db = client.dtr

        super(Store, self).__init__(store_id)

    def getName(self):

        return "flipkart"

    def scrapeStoreOrders(self,):

        orders = self._getActiveOrders()

    def scrapeAffiliate(self, startDate=None, endDate=None):

        br = getBrowserObject()

        br.open(AFFILIATE_URL)

        response = br.response()  # copy

        token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]

        data = {'__FK':token,

                'email':'saholic1@gmail.com',

                'password':'e8aacf6fc1e3998186a4a8e56e428f66'

        }

        br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))

        for status in [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_PENDING, AFF_STATUS_DISAPPROVED]:

            br.open(AFF_REPORT_URL % (status, '2015-01-02', '2015-02-01'))

            page = ungzipResponse(br.response())

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            try:

                tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]

            except:

                continue

            trElements = tableElement.findAll('tr')

            trElements.pop(0)

            for trElement in trElements:

                tdElements = trElement.findAll('td')

                productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]

                quantity =int(tdElements[3].text)

                price = int(float(tdElements[2].text))

                payOut = tdElements[6].text

                subTagId = tdElements[7].text

                reason = tdElements[9].text

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        br = getBrowserObject()

        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

        response = br.open(url)

        page = ungzipResponse(response)

        merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]

        print merchantOrderId

        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

        sections = soup.findAll("div", {"class":"ui-app-card-body"})

        sections.pop(1)

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        for data in sections:

            name = data.findAll("span")

            i=0

            while i< len(name):

                if "key" in str(name[i]):

                    if "Grand Total" in str(name[i]):

                        #Total Amount

                        merchantOrder.paidAmount = re.findall(r'\d+', name[i+1].text)[0]

                    elif "Order Date" in str(name[i]):

                        merchantOrder.placedOn = name[i+1].text 

                i=i+1    

        merchantOrder.merchantOrderId = merchantOrderId

        mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

        fkSubOrders = mainOrder.findAll("li")

        #remove unwanted list

        fkSubOrders.pop(-1)

        subOrders = []

        merchantOrder.subOrders = subOrders

        #fetching suborders details

        for subOrder in fkSubOrders:

            y = subOrder.find("header").findAll("span")

            for suborderId in y:

                if "value emp" in str(suborderId):

                    merchantSubOrderId = suborderId.text

                    break

            orderItems = subOrder.find("ul").findAll("div", recursive=False)

            i=0

            for orderItem in orderItems:

                merchantOrder.placedOn

                merchantSubOrderId = merchantSubOrderId + str(i)

                divs = orderItem.findAll('div', recursive=False)

                content = divs[0]

                orderTracking = divs[2]

                orderTrackingDet = divs[3]

                imgUrl = content.find('img')['src']

                lineDet = content.find("div",{"class":"product-info"})

                productTitle = lineDet.find("a",{"class":"product-title"}).text

                productUrl =  str(lineDet.find("a")['href'])

                start='pid='

                s=str(lineDet.find("a")['href'])

                productCode = re.findall(re.escape(start)+"(.*)",s)[0]

                mname = lineDet.findAll("span")

                k=0

                while k<len(mname):

                    if "note" in str(mname[k]):

                        if "Color:" in str(mname[k]):

                            productTitle = productTitle + " " + mname[k+1].text

                        elif "Qty:" in str(mname[k]):

                            quantity = int(mname[k+1].text)

                        elif "Subtotal:" in str(mname[k]):

                            amountPaid = re.findall(r'\d+', mname[k+1].text)[0]

                        elif "Delivery:" in str(mname[k]):

                            #Delivery Date for sub order

                            print "Delivery Date " +mname[k+1].text

                    k=k+1

                merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)

                merchantsubOrder.imgUrl  = imgUrl

                merchantsubOrder.productCode = productCode

                cashbackAmount = self.getCashbackAmount(subOrder.productCode, amountPaid)

                cashbackStatus = Store.CB_INIT

                if cashbackAmount <= 0:

                    cashbackStatus = Store.CB_NA

                merchantsubOrder.cashBackAmount = cashbackAmount

                merchantsubOrder.cashBackStatus = cashbackStatus      

                subOrders.append(merchantsubOrder)

                #To track shipping details         

                status=-1    

                tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                if "approveDetails-complete" in str(tr):

                    if "processingDetails-complete" in str(tr):

                        if "shippingDetails-complete" in str(tr):

                            if "delivery-complete" in str(tr):

                                status = MStore.ORDER_DELIVERED

                            else:     

                                status = MStore.ORDER_SHIPPED

                        else:

                            status = MStore.ORDER_PLACED

                    else: 

                        status = MStore.ORDER_PLACED

                elif str(tr) in ["approveDetails-ongoing"]:

                    status=MStore.ORDER_PLACED             

                if "dead" in str(tr):

                    status = MStore.ORDER_CANCELLED    

                print "Sub Order Status " + str(status)                      

                trackingDetailsActive = orderTrackingDet.find("div",{"class":"c-tabs-content m-top active"})

                print "Sub order tracking description " + trackingDetailsActive.text

        self._saveToOrder(todict(merchantOrder))

    def flipkartOrderTracking(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        br = getBrowserObject()

        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

        response = br.open(url)

        page = ungzipResponse(response, br)

        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})

        #fetching suborders details

        for subOrder in mainOrder:

            subOrd = subOrder.findAll("li")

            subOrd.pop(len(subOrd)-1)

            for productInfo in subOrd:

                status=-1    

                #To track shipping details         

                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})

                for trackingStatus in n:

                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                    if "approveDetails-complete" in str(tr):

                        print "Here"

                        if "processingDetails-complete" in str(tr):

                            print "There"

                            if "shippingDetails-complete" in str(tr):

                                print "Share"

                                if "delivery-complete" in str(tr):

                                    print "Last"

                                    status = 3

                                else:     

                                    status = 2

                            else:

                                status = 1 

                        else: 

                            status = 0     

                    if "dead" in str(tr):

                        status = 4    

                print status                      

def _saveToOrderFlipkart(self, order):

        collection = self.db.merchantOrder

        order = collection.insert(order)

def hex_md5(password):

    m = hashlib.md5()

    print(m.digest())

def main():

    store = getStore(2)

    #store.scrapeAffiliate()

    store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3019717576543723&token=14abca87835b66da4ce8fa9d7594d95f&src=or&pr=1')

    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')

    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')

    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')

    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')

    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')

    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')

    #hex_md5('spice@2020')

if __name__ == '__main__':

        main()

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj