WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/dtr/sources/amazon.py

 from base64 import encode
 from bs4 import BeautifulSoup
 from datetime import datetime
 from dtr.dao import Order, SubOrder
 from dtr.main import getStore, Store as MStore, ParseException, getBrowserObject, \
-    ungzipResponse
+    ungzipResponse, tprint
 from dtr.sources.flipkart import todict
 from paramiko import sftp
 from paramiko.client import SSHClient
 from paramiko.sftp_client import SFTPClient
+import mechanize
 import os.path
 import paramiko
 import re
 import time
 import traceback
 ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'
 ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'
 THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'
+AMAZON_AFF_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/listReports'
 class Store(MStore):
     orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],
                             MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],
                             MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],
                 resp['result'] = 'ORDER_NOT_CREATED'
                 return resp
     #This should be exposed from api for specific sources
     def scrapeStoreOrders(self):
-#        br = getBrowserObject()
+        br = getBrowserObject()
-#        orders = self.db.merchantOrder.find({"storeId":1, "closed":False, "subOrders.closed":False, "subOrders.trackingUrl":{"$exists":True}})
+        orders = self.db.merchantOrder.find({"storeId":1, "closed":False, "subOrders.closed":False, "subOrders.trackingUrl":{"$exists":True}})
-#        for merchantOrder in orders:
+        for merchantOrder in orders:
+            try:
+                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
+                closed = True
+                map1 = {}
-#            for subOrder in merchantOrder.get("subOrders"):
+                for subOrder in merchantOrder.get("subOrders"):
-#                trackingUrl = subOrder.get("trackingUrl")
+                    if subOrder.get("closed"):
+                        continue
-#                if not subOrder.get("closed") and trackingUrl is not None:
+                    elif subOrder.get("trackingUrl") is None:
+                        closed = False
-#
+                        continue
+                    findMap = {"orderId":merchantOrder.get("orderId"), "subOrders.merchantSubOrderId":subOrder.get("merchantSubOrderId")}
+                    trackingUrl = subOrder.get("trackingUrl")
+                    if not map1.has_key(trackingUrl):
-#                    self.parseTrackingUrl(br, trackingUrl)
+                        map1[trackingUrl] = self.parseTrackingUrl(br, trackingUrl)
+                    newOrder = map1.get(trackingUrl)
+                    newOrder['cashBackStatus'] = subOrder.get('cashBackStatus')
+                    updateMap = self.getUpdateMap(newOrder)
+                    print findMap, "\n", updateMap
+                    bulk.find(findMap).update({'$set' : updateMap})
+                    closed = closed and newOrder['closed']
+                if closed:
+                    print "Closed", {"orderId":merchantOrder.get("orderId")},  {"closed":True}
+                    bulk.find({"orderId":merchantOrder.get("orderId")}).update({"$set":{"closed":True}})
+                result = bulk.execute()
+                tprint(result)
-        pass
+            except:
+                tprint("Could not update " + str(merchantOrder['orderId']))
+                traceback.print_exc()
     def parseOldStlye(self, merchantOrder, soup):
         merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl
         table = soup.body.findAll("table", recursive=False)[1]
         #print table
         tables = table.tr.td.findAll("table", recursive=False)
         missingOrderUrls = []
         missingOrders = self._getMissingOrders({'userId':userId})
         for missingOrder in missingOrders:
             missingOrderUrls.append(ORDER_REDIRECT_URL%(missingOrder['merchantOrderId']))
-        orders = self._getActiveOrders({'userId':userId})
+        orders = self._getActiveOrders({'userId':userId, "$exists":{"subOrders.trackingUrl":False} })
         count = len(orders)
         print "count", count
         if count > 0:
             return missingOrderUrls + ['https://www.amazon.in/gp/css/order-history', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled', 'https://www.amazon.in/gp/css/order-history?orderFilter=cancelled&startIndex=10']
         else:
                 return key
         print "Detailed Status need to be mapped"
         print "Found new order status", detailedStatus
         raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
+    def scrapeAffiliate(self, startDate=None, endDate=None):
+        pass
     def parseTrackingUrl(self, br, trackingUrl):
+        subOrder = {}
         response = br.open(trackingUrl)
         page = ungzipResponse(response)
-        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
+        soup = BeautifulSoup(page)
-        alertContainer = soup.find("a-box-inner a-alert-container")
+        alertContainer = soup.find("div", {"class":"a-box-inner a-alert-container"})
+        if alertContainer is not None:
+            statusText = alertContainer.h4.span.text
+            detailedStatus = statusText.split(":")[0].strip()
+            subOrder['detailedStatus'] = detailedStatus
+            if detailedStatus.lower() == "in transit":
+                pass
+            elif detailedStatus.lower() == "undeliverable":
+                subOrder['status'] = MStore.ORDER_CANCELLED
+                return subOrder
+        else:
+            summaryLeft = soup.find(id="summaryLeft")
+            statusText = summaryLeft.h2.text
+            detailedStatus = statusText.split(":")[0].strip()
+            subOrder['detailedStatus'] = detailedStatus
+            if detailedStatus.lower() == "delivered":
+                subOrder['deliveredOn'] = summaryLeft.findAll("span")[-2].text.strip()
+                subOrder['status'] = MStore.ORDER_DELIVERED
+                return subOrder
+            else:
+                subOrder['expectedDelivery'] = summaryLeft.findAll("span")[-1].text.strip()
+        return subOrder
 def main():
     store = getStore(1)
-    #br = getBrowserObject()
+    br = mechanize.Browser()
-    #store.parseTrackingUrl(br, "http://www.amazon.in//gp/css/shiptrack/view.html/ref=oh_aui_direct_track_pkg_o00_s00?ie=UTF8&marketplaceOfOrigin=&orderID=171-7613541-7425906&orderingShipmentId=729710930106&packageId=1&ref=")
+    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
+                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),
+                     ('Accept-Encoding', 'gzip,deflate,sdch'),
+                     ('Accept-Language', 'en-US,en;q=0.8'),
+                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]
+    store.scrapeStoreOrders()
-    store.parseOrderRawHtml(2121, "13232", 14, readSSh("/home/amit/917.html"), "https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html/ref=ox_spc_place_order?ie=UTF8&hasWorkingJavascript=")
+    #store.parseOrderRawHtml(2121, "13232", 14, readSSh("/home/amit/917.html"), "https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html/ref=ox_spc_place_order?ie=UTF8&hasWorkingJavascript=")
 def readSSh(fileName):
     try:
             str1 = open(fileName).read()
             return str1
     except:

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/amazon.py – Rev 14608 → 14624