Subversion Repositories SmartDukaan

Rev

Rev 13690 | Rev 13730 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 13690 Rev 13721
Line 2... Line 2...
2 
Created on Jan 15, 2015
 2 
Created on Jan 15, 2015
3 
 
 3 
 
4 
@author: amit
 4 
@author: amit
5 
'''
 5 
'''
6 
from BeautifulSoup import BeautifulSoup
 6 
from BeautifulSoup import BeautifulSoup
- 
 
 7 
from datetime import datetime, date, timedelta
7 
from dtr import main
 8 
from dtr import main
8 
from dtr.dao import AffiliateInfo, Order, SubOrder
 9 
from dtr.dao import AffiliateInfo, Order, SubOrder
9 
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
 10 
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
10 
    Store as MStore, sourceMap
 11 
    Store as MStore, sourceMap
11 
from pprint import pprint
 12 
from pprint import pprint
Line 45... Line 46...
45 
    def getName(self):
 46 
    def getName(self):
46 
        return "flipkart"
 47 
        return "flipkart"
47
48
48 
    def scrapeStoreOrders(self,):
 49 
    def scrapeStoreOrders(self,):
49 
        orders = self._getActiveOrders()
 50 
        orders = self._getActiveOrders()
50
- 
 
51 
    def scrapeAffiliate(self, startDate=None, endDate=None):
 - 
 
52 
        br = getBrowserObject()
 51 
        br = getBrowserObject()
53 
        br.open(AFFILIATE_URL)
 52 
        for order in orders:
- 
 
 53
54 
        response = br.response()  # copy
 54 
            closed = True
55 
        token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
 55 
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
56 
        data = {'__FK':token,
 56 
            response = br.open(url)
57 
                'email':'saholic1@gmail.com',
 57 
            page = ungzipResponse(response)
58 
                'password':'e8aacf6fc1e3998186a4a8e56e428f66'
 - 
 
59 
        }
 58
60 
        br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
 - 
 
61 
        for status in [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_PENDING, AFF_STATUS_DISAPPROVED]:
 - 
 
62 
            br.open(AFF_REPORT_URL % (status, '2015-01-02', '2015-02-01'))
 - 
 
63 
            page = ungzipResponse(br.response())
 - 
 
64 
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
 59 
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
- 
 
 60
- 
 
 61 
            sections = soup.findAll("div", {"class":"ui-app-card-body"})
- 
 
 62 
            sections.pop(1)
- 
 
 63
- 
 
 64 
            mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
- 
 
 65 
            fkSubOrders = mainOrder.findAll("li")
- 
 
 66
- 
 
 67 
            #remove unwanted list
- 
 
 68 
            fkSubOrders.pop(-1)
- 
 
 69
- 
 
 70
- 
 
 71 
            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
- 
 
 72 
            #fetching suborders details
- 
 
 73 
            for subOrder in fkSubOrders:
- 
 
 74 
                updateMap = {}
- 
 
 75 
                y = subOrder.find("header").findAll("span")
- 
 
 76 
                for suborderId in y:
- 
 
 77 
                    if "value emp" in str(suborderId):
- 
 
 78 
                        merchantSubOrderId = suborderId.text
- 
 
 79 
                        break
- 
 
 80 
                orderItems = subOrder.find("ul").findAll("div", recursive=False)
- 
 
 81 
                i=0
- 
 
 82 
                for orderItem in orderItems:
- 
 
 83 
                    closedStatus = False
- 
 
 84 
                    merchantSubOrderId = merchantSubOrderId + str(i)
- 
 
 85 
                    subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
- 
 
 86 
                    if subOrder is None:
- 
 
 87 
                        break
- 
 
 88 
                    findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
- 
 
 89 
                    divs = orderItem.findAll('div', recursive=False)
- 
 
 90 
                    orderTracking = divs[2]
- 
 
 91 
                    orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})
- 
 
 92 
                    orderTrackingDet = orderTrackingDetDiv.div.header.div.text
- 
 
 93 
                    updateMap["subOrders.$.detailedStatus"] = orderTrackingDet
- 
 
 94 
                    tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
- 
 
 95
- 
 
 96 
                    cashbackStatus = subOrder.get("cashBackStatus")
- 
 
 97 
                    if "approveDetails-complete" in str(tr):
- 
 
 98 
                        if "processingDetails-complete" in str(tr):
- 
 
 99 
                            if "shippingDetails-complete" in str(tr):
- 
 
 100 
                                if "delivery-complete" in str(tr):
- 
 
 101 
                                    status = MStore.ORDER_DELIVERED
- 
 
 102 
                                    if cashbackStatus == Store.CB_INIT:
- 
 
 103 
                                        cashbackStatus = Store.CB_APPROVED
- 
 
 104 
                                    closedStatus = True
- 
 
 105 
                                else:
- 
 
 106 
                                    status = MStore.ORDER_SHIPPED
- 
 
 107 
                                    courierTrackAnchor = orderTrackingDetDiv.find('a')
- 
 
 108 
                                    trackingUrl = courierTrackAnchor("href")
- 
 
 109 
                                    trackingText = courierTrackAnchor.text
- 
 
 110 
                                    courierArr = trackingText.split(' : ')
- 
 
 111 
                                    updateMap["subOrders.$.trackingUrl"] = trackingUrl
- 
 
 112 
                                    updateMap["subOrders.$.trackingNumber"] = courierArr[1]
- 
 
 113 
                                    updateMap["subOrders.$.courierName"] = courierArr[0]
- 
 
 114 
                            else:
- 
 
 115 
                                status = MStore.ORDER_PLACED
- 
 
 116 
                        else:
- 
 
 117 
                            status = MStore.ORDER_PLACED
- 
 
 118 
                    elif str(tr) in ["approveDetails-ongoing"]:
- 
 
 119 
                        status=MStore.ORDER_PLACED
- 
 
 120 
                    if "dead" in str(tr):
- 
 
 121 
                        status = MStore.ORDER_CANCELLED
- 
 
 122 
                        closedStatus = True
- 
 
 123 
                        if cashbackStatus == Store.CB_INIT:
- 
 
 124 
                            cashbackStatus = Store.CB_CANCELLED
- 
 
 125
- 
 
 126 
                    updateMap["subOrders.$.cashBackStatus"] = cashbackStatus
- 
 
 127 
                    updateMap["subOrders.$.status"] = status
- 
 
 128 
                    updateMap["subOrders.$.closed"] = closedStatus
- 
 
 129 
                    if closed:
- 
 
 130 
                        closed = closedStatus
- 
 
 131 
                    bulk.find(findMap).update({'$set' : updateMap})
- 
 
 132 
            bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed}})
- 
 
 133 
            bulk.execute()
- 
 
 134
- 
 
 135 
    def scrapeAffiliate(self, startDate=None, endDate=None):
- 
 
 136 
        #get all yesterday's affiliate in pending or cancelled state and update to system
- 
 
 137 
        if datetime.now().hour == 1:
- 
 
 138 
            yesterdate = date.today() - timedelta(1)
- 
 
 139 
            syesterdate = yesterdate.strftime('%Y-%m-%d')
- 
 
 140 
            br = getBrowserObject()
- 
 
 141 
            br.open(AFFILIATE_URL)
- 
 
 142 
            response = br.response()  # copy
- 
 
 143 
            token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
- 
 
 144 
            data = {'__FK':token,
- 
 
 145 
                    'email':'saholic1@gmail.com',
- 
 
 146 
                    'password':'e8aacf6fc1e3998186a4a8e56e428f66'
- 
 
 147 
            }
- 
 
 148 
            br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
- 
 
 149 
            for status in [AFF_STATUS_PENDING, AFF_STATUS_CANCELLED, AFF_STATUS_DISAPPROVED]:
- 
 
 150 
                br.open(AFF_REPORT_URL % (status, syesterdate, syesterdate))
- 
 
 151 
                page = ungzipResponse(br.response())
- 
 
 152 
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
65 
            try:
 153 
                try:
66 
                tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]
 154 
                    tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]
67 
            except:
 155 
                except:
68 
                continue
 156 
                    continue
69 
            trElements = tableElement.findAll('tr')
 157 
                trElements = tableElement.findAll('tr')
70 
            trElements.pop(0)
 158 
                trElements.pop(0)
71 
            for trElement in trElements:
 159 
                for trElement in trElements:
72 
                tdElements = trElement.findAll('td')
 160 
                    tdElements = trElement.findAll('td')
73 
                productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]
 161 
                    productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]
74 
                quantity =int(tdElements[3].text)
 162 
                    quantity =int(tdElements[3].text)
75 
                price = int(float(tdElements[2].text))
 163 
                    price = int(re.findall(r'\d+', tdElements[2].text)[0])
76 
                payOut = tdElements[6].text
 164 
                    payOut = int(re.findall(r'\d+', tdElements[6].text)[0])
77 
                subTagId = tdElements[7].text
 165 
                    subTagId = tdElements[7].text
- 
 
 166 
                    affiliateInfo = AffiliateInfo(subTagId, self.store_id, status, None, syesterdate, payOut, None, None, price*quantity)
78 
                reason = tdElements[9].text
 167 
                    print todict(affiliateInfo)
- 
 
 168 
                    updateMap = {}
- 
 
 169 
                    updateMap['subOrders.$.unitPrice'] = price
- 
 
 170 
                    updateMap['subOrders.$.cashBackAmount'], updateMap['subOrders.$.cashBacPercentage'] = self.getCashbackAmount(productCode, price)
- 
 
 171 
                    self._updateOrdersPayBackStatus({'subTagId':subTagId, 'subOrders.productCode':productCode}, updateMap)
- 
 
 172
79
173
80 
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
 174 
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
81 
        br = getBrowserObject()
 175 
        br = getBrowserObject()
82 
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
 176 
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
83
177
Line 98... Line 192...
98 
            i=0
 192 
            i=0
99 
            while i< len(name):
 193 
            while i< len(name):
100 
                if "key" in str(name[i]):
 194 
                if "key" in str(name[i]):
101 
                    if "Grand Total" in str(name[i]):
 195 
                    if "Grand Total" in str(name[i]):
102 
                        #Total Amount
 196 
                        #Total Amount
103 
                        merchantOrder.paidAmount = re.findall(r'\d+', name[i+1].text)[0]
 197 
                        merchantOrder.paidAmount = int(re.findall(r'\d+', name[i+1].text)[0])
104 
                    elif "Order Date" in str(name[i]):
 198 
                    elif "Order Date" in str(name[i]):
105 
                        merchantOrder.placedOn = name[i+1].text
199 
                        merchantOrder.placedOn = name[i+1].text
106 
                i=i+1
200 
                i=i+1
107
201
108 
        merchantOrder.merchantOrderId = merchantOrderId
 202 
        merchantOrder.merchantOrderId = merchantOrderId
Line 156... Line 250...
156 
                            print "Delivery Date " +mname[k+1].text
 250 
                            print "Delivery Date " +mname[k+1].text
157 
                    k=k+1
 251 
                    k=k+1
158 
                merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)
 252 
                merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)
159 
                merchantsubOrder.imgUrl  = imgUrl
 253 
                merchantsubOrder.imgUrl  = imgUrl
160 
                merchantsubOrder.productCode = productCode
 254 
                merchantsubOrder.productCode = productCode
161
255 
                merchantsubOrder.merchantSubOrderId = merchantSubOrderId
162 
                cashbackAmount = self.getCashbackAmount(subOrder.productCode, amountPaid)
 256 
                cashbackAmount, cashbackPercent = self.getCashbackAmount(subOrder.productCode, amountPaid)
163 
                cashbackStatus = Store.CB_INIT
 257 
                cashbackStatus = Store.CB_INIT
164 
                if cashbackAmount <= 0:
 258 
                if cashbackAmount <= 0:
165 
                    cashbackStatus = Store.CB_NA
 259 
                    cashbackStatus = Store.CB_NA
166 
                merchantsubOrder.cashBackAmount = cashbackAmount
 260 
                merchantsubOrder.cashBackAmount = cashbackAmount
167 
                merchantsubOrder.cashBackStatus = cashbackStatus
261 
                merchantsubOrder.cashBackStatus = cashbackStatus
- 
 
 262 
                merchantsubOrder.cashBackPercentage = cashbackPercent
168
263
169 
                subOrders.append(merchantsubOrder)
 264 
                subOrders.append(merchantsubOrder)
170 
                #To track shipping details
265 
                #To track shipping details
171 
                status=-1
266 
                status=-1
172 
                tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
 267 
                tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
Line 175... Line 270...
175 
                        if "shippingDetails-complete" in str(tr):
 270 
                        if "shippingDetails-complete" in str(tr):
176 
                            if "delivery-complete" in str(tr):
 271 
                            if "delivery-complete" in str(tr):
177 
                                status = MStore.ORDER_DELIVERED
 272 
                                status = MStore.ORDER_DELIVERED
178 
                            else:
273 
                            else:
179 
                                status = MStore.ORDER_SHIPPED
 274 
                                status = MStore.ORDER_SHIPPED
- 
 
 275 
                                orderTracking.find('div', {'class':'tracking-remark active'})
180 
                        else:
 276 
                        else:
181 
                            status = MStore.ORDER_PLACED
 277 
                            status = MStore.ORDER_PLACED
182 
                    else:
278 
                    else:
183 
                        status = MStore.ORDER_PLACED
 279 
                        status = MStore.ORDER_PLACED
184 
                elif str(tr) in ["approveDetails-ongoing"]:
 280 
                elif str(tr) in ["approveDetails-ongoing"]:
Line 231... Line 327...
231 
                            status = 0
327 
                            status = 0
232 
                    if "dead" in str(tr):
 328 
                    if "dead" in str(tr):
233 
                        status = 4
329 
                        status = 4
234 
                print status
330 
                print status
235
331
236
- 
 
237
332
238 
def _saveToOrderFlipkart(self, order):
 333 
def _saveToOrderFlipkart(self, order):
239 
        collection = self.db.merchantOrder
 334 
        collection = self.db.merchantOrder
240 
        order = collection.insert(order)
 335 
        order = collection.insert(order)
241 
 
 336 
 
Line 247... Line 342...
247 
 
 342 
 
248 
def main():
 343 
def main():
249
344
250 
    store = getStore(2)
 345 
    store = getStore(2)
251 
    #store.scrapeAffiliate()
 346 
    #store.scrapeAffiliate()
252 
    store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3019717576543723&token=14abca87835b66da4ce8fa9d7594d95f&src=or&pr=1')
 347 
    #store.parseOrderRawHtml(12346, "SHA21423034609", 122324,  "html", 'https://www.flipkart.com/orderresponse?reference_id=OD0019889963104132&token=198b370dfc5a25c017b5982c369ee960&src=or&pr=1')
253 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
 348 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
254 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
 349 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
255 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
 350 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
256 
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
 351 
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
257 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
 352 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
258 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')
 353 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')
259 
    #hex_md5('spice@2020')
 354 
    #hex_md5('spice@2020')
- 
 
 355 
    store.scrapeStoreOrders()
260 
 
 356 
 
261 
 
 357 
 
262 
if __name__ == '__main__':
 358 
if __name__ == '__main__':
263 
        main()
 359 
        main()
264
360