WebSVN – SmartDukaan – Diff – //trunk/PyDTR/src/dtr/sources/shopclues.py

 '''
 from bs4 import BeautifulSoup
 from bson.binary import Binary
 from datetime import datetime, date, timedelta
 from dtr import main
-from dtr.dao import AffiliateInfo, Order, SubOrder
+from dtr.dao import AffiliateInfo, Order, SubOrder, ShopCluesAffiliateInfo
 from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
     Store as MStore, ungzipResponse, tprint
 from dtr.storage import Mongo
 from dtr.storage.Mongo import getImgSrc
 from dtr.utils.utils import fetchResponseUsingProxy
 import pymongo
 import re
 import time
 import traceback
 import urllib
+import urllib2
 from urlparse import urlparse, parse_qs
+import xml.etree.ElementTree as ET
+from dtr.storage import MemCache
 USERNAME='profittill2@gmail.com'
 PASSWORD='spice@2020'
-AFFILIATE_URL='http://affiliate.snapdeal.com'
-POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
+AFFLIATE_TRASACTIONS_URL = "https://admin.optimisemedia.com/v2/reports/affiliate/leads/leadsummaryexport.aspx?Contact=796881&Country=26&Agency=95&Merchant=420562&Status=-1&Year=%d&Month=%d&Day=%d&EndYear=%d&EndMonth=%d&EndDay=%d&DateType=0&Sort=CompletionDate&Login=1347562DA5E3EFF6FB1561765C47C782&Format=XML&RestrictURL=0"
 ORDER_TRACK_URL='http://www.shopclues.com/index.php?dispatch=order_lookup.details'
-CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'
+ORDER_TRACK_URL_DB='https://sm.shopclues.com/trackOrder?'
 BASE_URL= 'http://www.shopclues.com'
 BASE_MURL= 'http://m.shopclues.com'
 class Store(MStore):
                       MStore.ORDER_DELIVERED : ['delivered', 'complete'],
                       MStore.ORDER_SHIPPED : ['in transit', 'dispatched','shipped','order handed to courier'],
                       MStore.ORDER_CANCELLED : ['payment failed', 'canceled', 'payment declined', 'order on hold - cancellation requested by customer', 'courier returned']
+                      }
     OrderStatusConfirmationMap= {
-                                 "Payment Successful" : "P",
+                                 "P" : "Payment Successful",
-                                 "Order Declined" : "D",
+                                 "D" : "Order Declined",
-                                 "New Order - COD confirmation Pending" : "O"
+                                 "O" : "New Order - COD confirmation Pending"
+                                 }
     CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE
-    '''
-    def scrapeAffiliate(self, startDate=None, endDate=None):
-        br = getBrowserObject()
-        br.open(AFFILIATE_URL)
-        br.select_form(nr=0)
-        br.form['data[User][password]'] = PASSWORD
-        br.form['data[User][email]'] = USERNAME
-        br.submit()
-        response = br.open(CONFIG_URL)
-        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
-        print token
-        allOffers = self._getAllOffers(br, token)
-        self._saveToAffiliate(allOffers)
-    '''
+    def convertToObj(self,offer):
+        orderRef = offer['TransactionId']
+        orderRef = orderRef[0:len(orderRef)-10]
+        offer1 = ShopCluesAffiliateInfo(offer['UID'], offer['TransactionTime'], offer['TransactionId'], orderRef,  offer['MerchantRef'], offer['Merchant'], offer['PID'], offer['Product'], float(str(offer['SR'])), float(str(offer['TransactionValue'])), offer['UKey'], offer['ClickTime'], offer['Status'])
+        return offer1
+    def _saveToAffiliate(self, offers):
+        collection = self.db.shopcluesOrderAffiliateInfo
+        mcollection = self.db.merchantOrder
+        for offerObj in offers:
+            offer = self.convertToObj(offerObj)
+            collection.update({"transactionId":offer.transactionId, "subTagId":offer.subTagId, "payOut":offer.payOut},{"$set":todict(offer)}, upsert=True)
+            mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True}, {"$set":{"subOrders.$.missingAff":False}})
+    def scrapeAffiliate(self, startDate=datetime.datetime.today() - datetime.timedelta(days=10), endDate=datetime.datetime.today()):
+        uri = AFFLIATE_TRASACTIONS_URL%(startDate.year,startDate.month,startDate.day,endDate.year,endDate.month,endDate.day)
+        root = ET.parse(urllib2.urlopen(uri)).getroot()
+        if len(root)> 0 and len(root[0])> 0:
+            offers = []
+            for child in root[0][0]:
+                offers.append(child.attrib)
+            self._saveToAffiliate(offers)
     def _setLastSaleDate(self, saleDate):
         self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})
     def getName(self):
         return "shopclues"
             if detailedStatus.lower() in value:
                 return key
         print "Detailed Status need to be mapped", detailedStatus, self.store_id
         return None
-    def _getSingleSubOrderMap(self, orderId, orderStatus, statusTime, soup, subTagId):
+    def _getSingleSubOrderMap(self, orderId, soup, orderObj):
+        orderStatus = self.OrderStatusConfirmationMap.get(orderObj['0']['status'])
+        statusTime= soup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
         productDetailsMap = {}
         orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
-        orderTable.pop(0)
+        firstRow = orderTable.pop(0)
+        totalColumns = len(firstRow.find_all('td'))
-        productDetailsSubMap = {}
+        jsonSubOrdersMap = {}
+        count = 1
+        for val in orderObj['0']['items'].values():
+            newCount = 0
+            for key in jsonSubOrdersMap.keys():
+                splitKey = key.split('-')
+                if str(val['order_id']) == splitKey[0]:
+                    newCount = int(splitKey[1])
+                    break
+            if newCount >0 :
+                count = newCount +1
+                jsonSubOrdersMap[str(val['order_id'])+'-'+str(count)] = val
+            else:
+                jsonSubOrdersMap[str(val['order_id'])+'-'+str(count)] = val
+            count= count+1
+        count = 1
         for orderTr in orderTable:
+            productDetailsSubMap = None
+            productDetailsSubMap = {}
             cols = orderTr.find_all('td')
             product_details = cols[0].find_all('a')
             #print product_details
             productUrl = product_details[0].get('href')
             productName = product_details[0].contents[0].strip()
             quantity = int(cols[1].text.strip())
             sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
+            discount = 0
+            subtotal = 0
+            if totalColumns == 5:
+                if cols[3].text.strip()!='-' or 'Rs.' in cols[3].text.strip():
-            discount = long(cols[3].text.strip().replace("Rs.",""))
+                    discount = long(cols[3].text.strip().replace("Rs.",""))
+                else:
+                    discount = 0
-            subtotal = long(cols[4].text.strip().replace("Rs.",""))
+                subtotal = long(cols[4].text.strip().replace("Rs.",""))
+            else:
+                subtotal = long(cols[3].text.strip().replace("Rs.",""))
             productDetailsSubMap['productUrl']=BASE_MURL+productUrl
             productDetailsSubMap['productName']=productName
-            productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL+'&order_id=' +str(orderId)+'&email_id='+ subTagId.split('$')[1]
+            productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL_DB+'order_id=' +str(orderId)+'&email_id='+ orderObj['0']['email']
             productDetailsSubMap['sellingPrice']=sellingPrice
             productDetailsSubMap['quantity']=quantity
             productDetailsSubMap['discount']=discount
             productDetailsSubMap['subtotal']=subtotal
-            productDetailsSubMap['parentOrderId']=orderId
+            jsonSubOrderDetails = jsonSubOrdersMap.get(str(orderId)+'-'+str(count))
-            br = getBrowserObject()
-            productPage = br.open(BASE_MURL+productUrl)
-            productPageHeaders = str(productPage.info()).split('\n')
-            productPage = ungzipResponse(productPage)
-            jsonProductResponse = None
-            for header in productPageHeaders:
-                header = header.split(':')
-                if header[0] == 'Content-Type' and 'json' in header[1]:
-                    jsonProductResponse = json.loads(productPage)
-            productPageSoup= None
-            if jsonProductResponse is not None:
-                productPageSoup = BeautifulSoup(jsonProductResponse['text'])
-            else:
-                productPageSoup = BeautifulSoup(productPage)
-            productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
-            lengthProductCodeArr= len(productCodeArray)
-            productCode = productCodeArray[lengthProductCodeArr-2]
+            productCode = jsonSubOrderDetails['product_code']
-            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
-            productImgUrl = ''
-            if allproductImageTags is not None and len(allproductImageTags)>0:
+            productImgUrl = jsonSubOrderDetails['images']['image_path'][0]
-                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
             productDetailsSubMap['productCode']=productCode
             productDetailsSubMap['imgUrl']=productImgUrl
-        productDetailsSubMap['subOrderStatus']=orderStatus
+            productDetailsSubMap['subOrderStatus']=orderStatus
-        productDetailsSubMap['subOrderStatusTime']=statusTime
+            productDetailsSubMap['subOrderStatusTime']=statusTime
-        productDetailsMap[orderId]=productDetailsSubMap
+            productDetailsMap[str(orderId)+'-'+str(count)]=productDetailsSubMap
+            count = count +1
         return productDetailsMap
-    def _getMultiSubOrdersMap(self, orderId, soup):
+    def _getMultiSubOrdersMap(self, orderId, soup, orderObj):
         orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
-        orderTable.pop(0)
+        #firstRow = orderTable.pop(0)
+        #totalColumns = len(firstRow.find_all('td'))
         productDetailsMap = {}
+        '''
+        jsonSubOrdersMap = {}
+        for val in orderObj['0']['items'].values():
+            jsonSubOrdersMap[val['order_id']] = val
+        '''
+        existingOrders = []
         for orderTr in orderTable:
-            productDetailsSubMap = {}
+            subOrderDetailsMap = {}
             cols = orderTr.find_all('td')
             product_details = cols[0].find_all('a')
             #print product_details
             subOrderId= product_details[1].contents[0].strip()
+            if subOrderId in existingOrders:
+                continue
+            else:
+                existingOrders.append(subOrderId)
-            productUrl = product_details[0].get('href')
+            #productUrl = product_details[0].get('href')
-            productName = product_details[0].contents[0].strip()
+            #productName = product_details[0].contents[0].strip()
-            subOrderTrackingUrl = product_details[1].get('href')
+            subOrderTrackingParsingUrl = product_details[1].get('href')
+            #subOrderTrackingUrl = subOrderTrackingParsingUrl.split('order_lookup.details&')[1]
+            '''
             quantity = int(cols[1].text.strip())
             sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
+            discount = 0
+            subtotal = 0
+            if totalColumns == 5:
+                if cols[3].text.strip()!='-' or 'Rs.' in cols[3].text.strip():
-            discount = long(cols[3].text.strip().replace("Rs.",""))
+                    discount = long(cols[3].text.strip().replace("Rs.",""))
+                else:
+                    discount = 0
-            subtotal = long(cols[4].text.strip().replace("Rs.",""))
+                subtotal = long(cols[4].text.strip().replace("Rs.",""))
+            else:
+                subtotal = long(cols[3].text.strip().replace("Rs.",""))
             productDetailsSubMap['productUrl']=BASE_MURL+productUrl
             productDetailsSubMap['productName']=productName
-            productDetailsSubMap['subOrderTrackingUrl']=BASE_URL+subOrderTrackingUrl
+            productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL_DB+subOrderTrackingUrl
             productDetailsSubMap['sellingPrice']=sellingPrice
             productDetailsSubMap['quantity']=quantity
             productDetailsSubMap['discount']=discount
             productDetailsSubMap['subtotal']=subtotal
-            br = getBrowserObject()
-            productPage = br.open(BASE_MURL+productUrl)
-            productPageHeaders = str(productPage.info()).split('\n')
+            jsonSubOrderDetails = jsonSubOrdersMap.get(subOrderId)
-            productPage = ungzipResponse(productPage)
-            jsonProductResponse = None
-            for header in productPageHeaders:
-                header = header.split(':')
-                if header[0] == 'Content-Type' and 'json' in header[1]:
-                    jsonProductResponse = json.loads(productPage)
-            productPageSoup= None
-            if jsonProductResponse is not None:
-                productPageSoup = BeautifulSoup(jsonProductResponse['text'])
-            else:
-                productPageSoup = BeautifulSoup(productPage)
-            productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
-            lengthProductCodeArr= len(productCodeArray)
-            productCode = productCodeArray[lengthProductCodeArr-2]
+            productCode = jsonSubOrderDetails['product_code']
-            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
-            productImgUrl = ''
-            if allproductImageTags is not None and len(allproductImageTags)>0:
+            productImgUrl = jsonSubOrderDetails['images']['image_path'][0]
-                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
             productDetailsSubMap['productCode']=productCode
             productDetailsSubMap['imgUrl']=productImgUrl
+            '''
             br1 = getBrowserObject()
-            orderTrackingPage = br1.open(BASE_URL+subOrderTrackingUrl)
+            orderTrackingPage = br1.open(BASE_URL+'&'+subOrderTrackingParsingUrl)
             headers = str(orderTrackingPage.info()).split('\n')
             orderTrackingPage= ungzipResponse(orderTrackingPage)
             jsonResponse = None
             for header in headers:
                 header = header.split(':')
             orderTrackingPageSoup = None
             if jsonResponse is not None:
                 orderTrackingPageSoup = BeautifulSoup(str(jsonResponse['text']))
             else:
                 orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)
+            '''
             subOrderStatusList = orderTrackingPageSoup.findAll(attrs={'class' : 'price ord_status'})
             subOrderStatus = subOrderStatusList[0].contents[0].strip()
             subOrderStatusTime= orderTrackingPageSoup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
             productDetailsSubMap['subOrderStatus'] = subOrderStatus
             productDetailsSubMap['subOrderStatusTime'] = subOrderStatusTime
             productDetailsSubMap['parentOrderId']=orderId
+            '''
+            subOrderDetailsMap = self._getSingleSubOrderMap(subOrderId, orderTrackingPageSoup, orderObj)
-            productDetailsMap[subOrderId]=productDetailsSubMap
+            productDetailsMap = dict(productDetailsMap.items()+subOrderDetailsMap.items())
         return productDetailsMap
     def updateCashbackInSubOrders(self, subOrders):
         for subOrder in subOrders:
             subOrder.cashBackStatus = cashbackStatus
             subOrder.cashBackAmount = cashbackAmount
             subOrder.cashBackPercentage = percentage
         return subOrders
-    def _parseOrders(self, orderId, subTagId, userId, page, orderSuccessUrl):
+    def _parseOrders(self, orderId, subTagId, userId, page, orderSuccessUrl, orderObj):
         soup = BeautifulSoup(page)
         productDetailsMap = {}
-        orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})
         paymentFields = soup.findAll(attrs={'class' : 'box_paymentcalculations_row'})
         orderDateList = soup.findAll(attrs={'class':'price ord_date'})
         placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("  ","")
         ordersubtotal=0
         ordercluebucks=0
                             print ordershippingcost
                         if 'Clue' in value.text.strip():
                             ordercluebucks = long(val.contents[3].text.strip().replace("Rs.",""))
                             print ordercluebucks
                         if 'Total' in value.text.strip():
-                            ordertotal = long(val.contents[3].text.strip().replace("Rs.",""))
+                            ordertotal = val.contents[3].text.strip().replace("Rs.","")
+                            ordertotal = ordertotal.replace(',','')
                             print ordertotal
-        if orderStatusList is not None and len(orderStatusList)>0:
+        if orderObj['0']['is_parent_order'] == 'N':
-            orderStatus = orderStatusList[0].contents[0].strip()
-            statusTime= soup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
-            productDetailsMap = self._getSingleSubOrderMap(orderId, orderStatus, statusTime, soup, subTagId)
+            productDetailsMap = self._getSingleSubOrderMap(orderId, soup, orderObj)
         else:
-            productDetailsMap = self._getMultiSubOrdersMap(orderId, soup)
+            productDetailsMap = self._getMultiSubOrdersMap(orderId, soup, orderObj)
-        merchantOrder = Order(orderId, userId, subTagId.split('$')[0], self.store_id, orderSuccessUrl)
+        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
         merchantOrder.placedOn = placedOn
-        merchantOrder.merchantOrderId = orderId
+        merchantOrder.merchantOrderId = orderObj['0']['order_id']
         merchantOrder.paidAmount = ordertotal
         totalOrdersAmount = 0
         totalDiscount = 0
         subOrders= []
         for key in productDetailsMap:
             subOrder.offerDiscount = subOrderDetail['discount'] * subOrderDetail['quantity']
             subOrder.unitPrice = subOrderDetail['sellingPrice']
             subOrder.productCode = subOrderDetail['productCode']
             subOrder.amountPaid = subOrderDetail['subtotal']
             subOrder.quantity = subOrderDetail['quantity']
+            subOrder.tracingkUrl = subOrderDetail['subOrderTrackingUrl']
             subOrders.append(subOrder)
         print totalOrdersAmount, totalDiscount
         merchantOrder.totalAmount = totalOrdersAmount
         merchantOrder.discountApplied = totalDiscount
         merchantOrder.deliveryCharges = ordershippingcost
         merchantOrder.subOrders = self.updateCashbackInSubOrders(subOrders)
         return merchantOrder
     def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
-                    #print merchantOrder
         resp = {}
         try:
+            rawHtmlSoup = BeautifulSoup(rawHtml)
+            emailId = None
+            orderObj = None
+            merchantOrderId = None
+            for script in rawHtmlSoup.find_all('script'):
+                if 'orderConfirmation' in script.text:
+                    orderObj = json.loads(script.text.strip().split('\n')[0].split('orderConfirmation = ')[1].split(';')[0])
+                    if merchantOrderId is None:
+                        merchantOrderId = orderObj['0']['order_id']
+                        emailId = orderObj['0']['email']
             br = getBrowserObject()
-            url = ORDER_TRACK_URL +'&order_id=' +str(orderId)+'&email_id='+ subTagId.split('$')[1]
+            url = ORDER_TRACK_URL +'&order_id=' +str(merchantOrderId)+'&email_id='+ emailId
             page = br.open(url)
             headers = str(page.info()).split('\n')
             page = ungzipResponse(page)
             jsonResponse = None
             for header in headers:
                 if header[0] == 'Content-Type' and 'json' in header[1]:
                     jsonResponse = json.loads(page)
             if jsonResponse is not None:
                 page = jsonResponse['text']
-            merchantOrder = self._parseOrders(orderId, subTagId, userId, page, orderSuccessUrl)
+            merchantOrder = self._parseOrders(orderId, subTagId, userId, page, orderSuccessUrl, orderObj)
-            merchantOrder.orderTrackingUrl = url
+            merchantOrder.orderTrackingUrl = ORDER_TRACK_URL_DB + 'order_id=' +str(orderId)+'&email_id='+ emailId
             if self._saveToOrder(todict(merchantOrder)):
                 resp['result'] = 'ORDER_CREATED'
             else:
                 resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
         except:
             print "Error occurred"
             traceback.print_exc()
             resp['result'] = 'ORDER_NOT_CREATED'
-    def parseSingleSubOrder(self, soup, emailId, subOrderId, subOrderStatus):
+    def parseSingleSubOrder(self, soup, emailId, subOrderId):
+        orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})
+        subOrderStatus = orderStatusList[0].contents[0].strip()
         orderDateList = soup.findAll(attrs={'class':'price ord_date'})
         placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("  ","")
         orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
-        orderTable.pop(0)
+        firstRow = orderTable.pop(0)
+        totalColumns = len(firstRow.find_all('td'))
         subOrders =[]
+        count =1
         for orderTr in orderTable:
             cols = orderTr.find_all('td')
             product_details = cols[0].find_all('a')
             #print product_details
             productUrl = product_details[0].get('href')
             productName = product_details[0].contents[0].strip()
             quantity = int(cols[1].text.strip())
             sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
+            discount = 0
+            subtotal = 0
+            if totalColumns == 5:
+                if cols[3].text.strip()!='-' or 'Rs.' in cols[3].text.strip():
-            discount = long(cols[3].text.strip().replace("Rs.",""))
+                    discount = long(cols[3].text.strip().replace("Rs.",""))
+                else:
+                    discount = 0
-            subtotal = long(cols[4].text.strip().replace("Rs.",""))
+                subtotal = long(cols[4].text.strip().replace("Rs.",""))
+            else:
+                subtotal = long(cols[3].text.strip().replace("Rs.",""))
             br = getBrowserObject()
             productPage = br.open(BASE_MURL+productUrl)
             productPageHeaders = str(productPage.info()).split('\n')
             productPage = ungzipResponse(productPage)
             jsonProductResponse = None
             productPageSoup= None
             if jsonProductResponse is not None:
                 productPageSoup = BeautifulSoup(jsonProductResponse['text'])
             else:
                 productPageSoup = BeautifulSoup(productPage)
-            productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
-            lengthProductCodeArr= len(productCodeArray)
-            productCode = productCodeArray[lengthProductCodeArr-2]
+            productCode = productPageSoup.find('input', {'type':'hidden'})['value']
             allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
             productImgUrl = ''
             if allproductImageTags is not None and len(allproductImageTags)>0:
                 productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
             subOrder = SubOrder(productName, productUrl, placedOn, subtotal)
-            subOrder.merchantSubOrderId = subOrderId
+            subOrder.merchantSubOrderId = str(subOrderId)+'-'+str(count)
             subOrder.detailedStatus = subOrderStatus
             subOrder.imgUrl = productImgUrl
             subOrder.offerDiscount = discount*quantity
             subOrder.unitPrice = sellingPrice
             subOrder.productCode = productCode
             subOrder.amountPaid = subtotal
             subOrder.quantity = quantity
+            subOrder.tracingkUrl = ORDER_TRACK_URL_DB +  'order_id=' +subOrderId+'&email_id='+ emailId
             subOrders.append(subOrder)
+            count = count +1
         subOrders = self.updateCashbackInSubOrders(subOrders)
-        return subOrders[0]
+        return subOrders
-    def parseMultiSubOrders(self, soup):
+    def parseMultiSubOrders(self, soup, emailId):
         orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
+        '''
-        orderTable.pop(0)
+        firstRow = orderTable.pop(0)
+        totalColumns = len(firstRow.find_all('td'))
         orderDateList = soup.findAll(attrs={'class':'price ord_date'})
         placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("  ","")
+        '''
         subOrders = []
+        existingOrders = []
         for orderTr in orderTable:
-            productDetailsSubMap = {}
             cols = orderTr.find_all('td')
             product_details = cols[0].find_all('a')
             #print product_details
             subOrderId= product_details[1].contents[0].strip()
+            if subOrderId in existingOrders:
+                continue
+            else:
+                existingOrders.append(subOrderId)
+            '''
             productUrl = product_details[0].get('href')
             productName = product_details[0].contents[0].strip()
+            '''
-            subOrderTrackingUrl = product_details[1].get('href')
+            subOrderTrackingParsingUrl = product_details[1].get('href')
+            #subOrderTrackingUrl = subOrderTrackingParsingUrl.split('order_lookup.details&')[1]
+            '''
             quantity = int(cols[1].text.strip())
             sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
+            discount = 0
+            subtotal = 0
+            if totalColumns == 5:
+                if cols[3].text.strip()!='-' or 'Rs.' in cols[3].text.strip():
-            discount = long(cols[3].text.strip().replace("Rs.",""))
+                    discount = long(cols[3].text.strip().replace("Rs.",""))
+                else:
+                    discount = 0
-            subtotal = long(cols[4].text.strip().replace("Rs.",""))
+                subtotal = long(cols[4].text.strip().replace("Rs.",""))
+            else:
+                subtotal = long(cols[3].text.strip().replace("Rs.",""))
             br = getBrowserObject()
             productPage = br.open(BASE_MURL+productUrl)
             productPageHeaders = str(productPage.info()).split('\n')
             productPage = ungzipResponse(productPage)
             jsonProductResponse = None
             productPageSoup= None
             if jsonProductResponse is not None:
                 productPageSoup = BeautifulSoup(jsonProductResponse['text'])
             else:
                 productPageSoup = BeautifulSoup(productPage)
-            productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
-            lengthProductCodeArr= len(productCodeArray)
-            productCode = productCodeArray[lengthProductCodeArr-2]
+            productCode = productPageSoup.find('input', {'type':'hidden'})['value']
             allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
             productImgUrl = ''
             if allproductImageTags is not None and len(allproductImageTags)>0:
                 productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
             productDetailsSubMap['productCode']=productCode
             productDetailsSubMap['imgUrl']=productImgUrl
+            '''
             br1 = getBrowserObject()
-            orderTrackingPage = br1.open(BASE_URL+subOrderTrackingUrl)
+            orderTrackingPage = br1.open(BASE_URL+'&'+subOrderTrackingParsingUrl)
             headers = str(orderTrackingPage.info()).split('\n')
             orderTrackingPage= ungzipResponse(orderTrackingPage)
             jsonResponse = None
             for header in headers:
                 header = header.split(':')
             orderTrackingPageSoup = None
             if jsonResponse is not None:
                 orderTrackingPageSoup = BeautifulSoup(str(jsonResponse['text']))
             else:
                 orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)
-            subOrderStatusList = orderTrackingPageSoup.findAll(attrs={'class' : 'price ord_status'})
+            subOrdersDetails = self.parseSingleSubOrder(orderTrackingPageSoup, emailId, subOrderId)
-            subOrderStatus = subOrderStatusList[0].contents[0].strip()
+            subOrders = list(set(subOrders + subOrdersDetails))
-            subOrderStatusTime= orderTrackingPageSoup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
-            subOrder = SubOrder(productName, productUrl, placedOn, subtotal)
-            subOrder.merchantSubOrderId = subOrderId
-            subOrder.detailedStatus = subOrderStatus
-            subOrder.imgUrl = productImgUrl
-            subOrder.offerDiscount = discount*quantity
-            subOrder.unitPrice = sellingPrice
-            subOrder.productCode = productCode
-            subOrder.amountPaid = subtotal
-            subOrder.quantity = quantity
-            subOrders.append(subOrder)
+        return subOrders
-            return self.updateCashbackInSubOrders(subOrders)
     def scrapeStoreOrders(self,):
         #collectionMap = {'palcedOn':1}
         searchMap = {}
         collectionMap = {"orderTrackingUrl":1}
         orders = self._getActiveOrders(searchMap,collectionMap)
         for order in orders:
             print "Order", self.store_name, order['orderId'], order['orderTrackingUrl']
-            url = order['orderTrackingUrl']
+            url = ORDER_TRACK_URL + order['orderTrackingUrl'].split('trackOrder?')[1]
             page = fetchResponseUsingProxy(url)
             soup = BeautifulSoup(page)
             bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
             closed = True
             orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})
             if orderStatusList is not None and len(orderStatusList)>0:
                 subOrderId = soup.findAll(attrs={'class':'price ord_no'})[0].text.strip()
-                subOrder =  self._isSubOrderActive(order, subOrderId)
                 orderStatus = orderStatusList[0].contents[0].strip()
+                orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
-                if subOrder is None:
+                orderTable.pop(0)
-                    try:
+                count = 1
-                        subOrder = self.parseSingleSubOrder(soup, order['orderTrackingUrl'].split('email_id=')[1], subOrderId, orderStatus)
-                        if subOrder is None:
+                while count <= len(orderTable):
-                            continue
-                        self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrder])}}})
-                        print "Added new suborders to Order id - ", order['orderId']
+                    subOrder =  self._isSubOrderActive(order, str(subOrderId)+'-'+str(count))
-                        closed = False
+                    count = count +1
-                    except:
+                    if subOrder is None:
-                        pass
+                        try:
+                            subOrders = self.parseSingleSubOrder(soup, order['orderTrackingUrl'].split('email_id=')[1], subOrderId)
-                    continue
+                            for subOrder in subOrders:
-                elif subOrder['closed']:
+                                if subOrder is None:
-                    continue
+                                    continue
-                else:
-                    findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}
+                                self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrder])}}})
-                    updateMap = {}
-                    updateMap["subOrders.$.detailedStatus"] = orderStatus
+                                print "Added new suborders to Order id - ", order['orderId']
-                    status = self._getStatusFromDetailedStatus(orderStatus)
+                                closed = False
-                    closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
-                    if status is not None:
+                        except:
-                        updateMap["subOrders.$.status"] = status
+                            pass
-                    if closedStatus:
+                        continue
-                        #if status is closed then change the paybackStatus accordingly
-                        updateMap["subOrders.$.closed"] = True
+                    elif subOrder['closed']:
-                        if status == Store.ORDER_DELIVERED:
+                        continue
-                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
-                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
-                        elif status == Store.ORDER_CANCELLED:
-                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
-                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
                     else:
+                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}
+                        updateMap = {}
+                        updateMap["subOrders.$.detailedStatus"] = orderStatus
+                        status = self._getStatusFromDetailedStatus(orderStatus)
+                        closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
+                        if status is not None:
+                            updateMap["subOrders.$.status"] = status
+                        if closedStatus:
+                            #if status is closed then change the paybackStatus accordingly
+                            updateMap["subOrders.$.closed"] = True
+                            if status == Store.ORDER_DELIVERED:
+                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:
+                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
+                            elif status == Store.ORDER_CANCELLED:
+                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:
+                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
+                        else:
-                        closed = False
+                            closed = False
-                    bulk.find(findMap).update({'$set' : updateMap})
+                        bulk.find(findMap).update({'$set' : updateMap})
             else:
-                subOrdersList = self.parseMultiSubOrders(soup)
+                subOrdersList = self.parseMultiSubOrders(soup, order['orderTrackingUrl'].split('email_id=')[1])
                 for subOrderObj in subOrdersList:
                     subOrder =  self._isSubOrderActive(order, subOrderObj.merchantSubOrderId)
                     if subOrder is None:
                         self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrderObj])}}})
                         print "Added new suborders to Order id - ", order['orderId']
                             closed = False
                         bulk.find(findMap).update({'$set' : updateMap})
             bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
             result = bulk.execute()
             tprint(result)
+    '''
+    def getTrackingUrls(self, userId):
+        missingOrderUrls = ['https://sm.shopclues.com/myaccount','https://sm.shopclues.com/myorders']
+        return missingOrderUrls
-    def saveToAffiliate(self, offers):
+    def parseMyProfileForEmailId(self, userId, url, rawhtml):
+        profileSoup = BeautifulSoup(rawhtml)
+        if profileSoup.find('input', {'name':'user_email'}) is not None:
+            emailId = profileSoup.find('input', {'name':'user_email'})['value']
+            if emailId is not None and emailId.strip() !='':
-        raise NotImplementedError
+                mc = MemCache()
+                mc.set(str(userId), emailId, 600)
+                return 'EMAIL_SET_SUCCESS'
+            else:
+                return 'EMAIL_NOT_FOUND'
+        else:
+            return 'EMAIL_NOT_FOUND'
+    def parseMyOrdersForEmailId(self, userId, url, rawhtml):
+        myOrdersPageSoup = BeautifulSoup(rawhtml)
+        if myOrdersPageSoup.find('a', {'class':'detail'}) is not None:
+            emailId = myOrdersPageSoup.find('a', {'class':'detail'})['href'].split('&')[1].split('=')[1]
+            mc = MemCache()
+            mc.set(str(userId), emailId, 600)
+            return 'EMAIL_SET_SUCCESS'
+        else:
+            return 'EMAIL_NOT_FOUND'
+    '''
 def todict(obj, classkey=None):
     if isinstance(obj, dict):
         data = {}
         for (k, v) in obj.items():
             if not callable(value) and not key.startswith('_')])
         if classkey is not None and hasattr(obj, "__class__"):
             data[classkey] = obj.__class__.__name__
         return data
     else:
         return obj

Subversion Repositories SmartDukaan

(root)//trunk/PyDTR/src/dtr/sources/shopclues.py – Rev 15510 → 15791