WebSVN – SmartDukaan – /trunk/PyDTR/src/dtr/sources/amazon.py

# coding=utf-8
'''
Created on Jan 15, 2015

@author: amit
'''
from base64 import encode
from bs4 import BeautifulSoup
from datetime import datetime
from dtr.dao import Order, SubOrder
from dtr.main import getStore, Store as MStore, ParseException
from dtr.sources.flipkart import todict
from paramiko import sftp
from paramiko.client import SSHClient
from paramiko.sftp_client import SFTPClient
import os.path
import paramiko
import re
import time
import traceback

ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'
ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'
THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'
class Store(MStore):
    
    orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],
                            MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],
                            MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],
                            MStore.ORDER_DELIVERED : ['delivered']
                           }

    def __init__(self,store_id):
        super(Store, self).__init__(store_id)
        
    def getName(self):
        return "flipkart"
    
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl, insert=False):
        resp = {}
        if ORDER_SUCCESS_URL in orderSuccessUrl or THANKYOU_URL in orderSuccessUrl:
            try:
                soup = BeautifulSoup(rawHtml)
                orderUrl = soup.find('div', {"id":"thank-you-box-wrapper"}).div.findAll('div', recursive=False)[1].a['href']
                merchantOrderId = re.findall(r'.*&oid=(.*)&?.*?', orderUrl)[0]
                order = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl, False)
                order.orderSuccessUrl = ORDER_REDIRECT_URL % (merchantOrderId)
                order.merchantOrderId = merchantOrderId
                order.requireDetail = True
                order.closed = None
                if self._saveToOrder(todict(order)):
                    resp['result'] = 'ORDER_CREATED'
                    resp["url"] = ORDER_REDIRECT_URL % (merchantOrderId)
                    resp["htmlRequired"] = True
                    resp['orderId'] = orderId
                else:
                    resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                    
                return resp
            except:
                resp["result"] = 'ORDER_NOT_CREATED'
                return resp
        
        else:
            try:
                merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
                merchantOrder.orderTrackingUrl = orderSuccessUrl
                soup = BeautifulSoup(rawHtml,from_encoding="utf-8")
                
                table = soup.body.findAll("table", recursive=False)[1]
                #print table
                tables = table.tr.td.findAll("table", recursive=False)
                for tr in tables[2].findAll("tr"):
                    boldElement = tr.td.b
                    if "Order Placed" in str(boldElement):
                        merchantOrder.placedOn = boldElement.next_sibling.strip()
                    if "order number" in str(boldElement):
                        merchantOrder.merchantOrderId = boldElement.next_sibling.strip()
                    if "Order Total" in str(boldElement):
                        merchantOrder.paidAmount = int(float(boldElement.find('span').contents[-1].replace(',','')))
                anchors = table.tr.td.findAll("a", recursive=False)
                paymentAnchor = anchors.pop(-1)
                
                count = 0
                subOrders = []
                merchantOrder.subOrders = subOrders
                for anchor in anchors:
                    count += 1
                    tab = anchor.next_sibling
                    status = MStore.ORDER_PLACED
                    subStr = "Delivery #" + str(count) + ":"
                    if subStr in  tab.find("b").text:
                        detailedStatus = tab.find("b").text.replace(subStr, '').strip()
                    
                    tab = tab.next_sibling.next_sibling
                    trs = tab.find("table").find('tbody').findAll("tr", recursive = False)
                    
                    estimatedDelivery = trs[0].td.find("b").next_sibling.strip()
                    
                    orderItemTrs = trs[1].findAll("td", recursive=False)[1].table.tbody.findAll("tr", recursive = False)
                    i = -1
                    for orderItemTr in orderItemTrs:
                        i += 1
                        if i%2 == 0:
                            continue
                        quantity =  int(re.findall(r'\d+', orderItemTr.td.contents[0])[0])
                        
                        productUrl = orderItemTr.td.contents[1].a["href"]
                        productTitle = orderItemTr.td.contents[1].a.text
                        
                        unitPrice = int(float(orderItemTr.findAll('td')[1].span.text.replace('Rs. ','').replace(',','')))
                        
                    
                        subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, unitPrice*quantity, status, quantity)
                        subOrder.merchantSubOrderId = orderItemTr.find("input")["name"]
                        subOrder.estimatedDeliveryDate = estimatedDelivery
                        subOrder.productCode = productUrl.split('/')[5]
                        subOrder.detailedStatus = detailedStatus
                        (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, unitPrice*quantity)
                        cashbackStatus = Store.CB_PENDING
                        if cashbackAmount <= 0:
                            cashbackStatus = Store.CB_NA
                        subOrder.cashBackStatus = cashbackStatus
                        subOrder.cashBackAmount = cashbackAmount
                        if percentage > 0:
                            subOrder.cashBackPercentage = percentage
                        subOrders.append(subOrder)
                priceList = paymentAnchor.next_sibling.next_sibling.next_sibling.table.table.tbody.tbody.tbody.findAll('tr', recursive=False)
                totalAmount = 0
                grandAmount = 0
                for price in priceList:
                    labelTd = price.td
                    if 'Subtotal:' in labelTd.text:
                        totalAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))
                    elif 'Grand Total:' in labelTd.text:
                        grandAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))
                if grandAmount < totalAmount:
                    diff = totalAmount - grandAmount
                    for subOrder in merchantOrder.subOrders:
                        subOrder.amountPaid -= int(diff*(1-subOrder.amountPaid/totalAmount))
                self._updateToOrder(todict(merchantOrder))
                resp['result'] = 'ORDER_CREATED'
                return resp    
            except:
                print "Error occurred"
                traceback.print_exc()
                resp['result'] = 'ORDER_NOT_CREATED'
                return resp    
                    
    #This should be exposed from api for specific sources
    def scrapeStoreOrders(self):
        pass
    
    def getTrackingUrls(self, userId):
        
        missingOrderUrls = []
        missingOrders = self._getMissingOrders({'userId':userId})
        for missingOrder in missingOrders:
            missingOrderUrls.append(ORDER_REDIRECT_URL%(missingOrder['merchantOrderId']))
        orders = self._getActiveOrders({'userId':userId})
        count = len(orders)
        print "count", count
        if count > 0:
            return missingOrderUrls + ['https://www.amazon.in/gp/css/order-history', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled', 'https://www.amazon.in/gp/css/order-history?orderFilter=cancelled&startIndex=10']
        else: 
            return missingOrderUrls
            
    def trackOrdersForUser(self, userId, url, rawHtml):
        directory = "/tmp/User" + str(userId)
        if not os.path.exists(directory):
            os.makedirs(directory)
        filename = directory + "/" + str(datetime.now())
        print "filename---", filename
        f = open(filename,'w')
        f.write(rawHtml) # python will convert \n to os.linesep
        f.close() # you can omit in most cases as the destructor will call if
        
        try:
            searchMap = {'userId':userId}
            collectionMap = {'merchantOrderId':1}
            activeOrders = self._getActiveOrders(searchMap, collectionMap)
            datetimeNow = datetime.now()
            timestamp = int(time.mktime(datetimeNow.timetuple()))
            print "url----------------", url
            
            if url == 'https://www.amazon.in/gp/css/order-history' or 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled' in url:
                soup = BeautifulSoup(rawHtml)
                allOrders = soup.find(id="ordersContainer").findAll('div', {'class':'a-box-group a-spacing-base order'})
                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
                for activeOrder in activeOrders:
                    for orderEle in allOrders:
                        orderdiv = orderEle.find('div', {'class':'a-box a-color-offset-background order-info'}).find('div', {'class':'a-fixed-right-grid-col actions a-col-right'})
                        merchantOrderId = orderdiv.find('span', {'class':'a-color-secondary value'}).text.strip()
                        if merchantOrderId==activeOrder['merchantOrderId']:
                            closed = True
                            shipments = orderEle.findAll('div',{'class':re.compile('.*?a-box.*?')}, recursive=False)
                            shipments.pop(0)
                            for shipment in shipments:
                                shipdiv = shipment.find('div', {'class':'a-box-inner'})
                                sdivs = shipment.div.div.findAll('div', recursive=False)
                                orderStatus = sdivs[0].span.text.strip()
                                status = self._getStatusFromDetailedStatus(orderStatus)
                                if status == MStore.ORDER_DELIVERED:
                                    deliveredOn = sdivs[0].findAll('span')[-1].text.strip()
                                    deliveredOn = deliveredOn.split(":")[1].strip()
                                deliveryestimatespan = sdivs[0].find('span', {'class':'a-color-success'})
                                deliveryEstimate = None
                                if deliveryestimatespan is not None:
                                    deliveryEstimate = deliveryestimatespan.find('span', {'class':'a-text-bold'}).text.strip()
                                productDivs = shipdiv.find('div', {'class':re.compile('.*?a-spacing-top-medium.*?')}).find('div', {'class':'a-row'}).findAll('div', recursive=False)
                                trackingUrl = None
                                for buttonDiv in shipdiv.findAll('span', {'class':'a-button-inner'}):
                                    if buttonDiv.find('a').text.strip()=='Track package':
                                        trackingUrl = buttonDiv.find('a')['href'].strip()
                                        if not trackingUrl.startswith("http"):
                                            trackingUrl = "https://www.amazon.in/" + trackingUrl
                                        break
                                for prodDiv in productDivs:
                                    prodDiv.find('div', {'class':'a-fixed-left-grid-inner'})
                                    productTitle = prodDiv.find('div', {'class':'a-fixed-left-grid-inner'}).find("div", {'class':'a-row'}).find('a').text.strip()
                                    imgUrl = prodDiv.find("img")["src"]
                                    for subOrder in activeOrder['subOrders']:
                                        if subOrder['productTitle'] == productTitle:
                                            findMap = {"orderId": activeOrder['orderId'], "subOrders.merchantSubOrderId": subOrder.get("merchantSubOrderId")}
                                            updateMap = {}
                                            closedStatus = False
                                            updateMap['subOrders.$.imgUrl'] = imgUrl
                                            updateMap['subOrders.$.lastTracked'] = timestamp
                                            updateMap['subOrders.$.detailedStatus'] = orderStatus
                                            cashbackStatus = subOrder.get("cashBackStatus")
                                            updateMap['subOrders.$.status'] = status 
                                        
                                            if status==MStore.ORDER_DELIVERED:                               
                                                updateMap['subOrders.$.deliveredOn'] = deliveredOn
                                                closedStatus = True
                                                updateMap['subOrders.$.closed'] = True
                                                if cashbackStatus == Store.CB_PENDING:
                                                    updateMap['subOrders.$.cashbackStatus'] = Store.CB_APPROVED
                                            if status==MStore.ORDER_CANCELLED:     
                                                closedStatus = True
                                                updateMap['subOrders.$.closed'] = True
                                                if cashbackStatus == Store.CB_PENDING:
                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_CANCELLED
                                            if status==MStore.ORDER_SHIPPED:   
                                                updateMap['subOrders.$.estimatedDeliveryDate'] = deliveryEstimate
                                                if trackingUrl is not None:
                                                    updateMap['subOrders.$.trackingUrl'] = trackingUrl
                                            if not closedStatus:
                                                closed = False
                                            bulk.find(findMap).update({'$set' : updateMap})
                                            break
                            bulk.find({'orderId': activeOrder['orderId']}).update({'$set':{'closed': closed}})            
                bulk.execute()
                return 'PARSED_SUCCESS'
            else:
                merchantOrderId = re.findall(r'https://www.amazon.in/gp/css/summary/edit.html\?orderID=(.*)?', url, re.IGNORECASE)[0] 
                merchantOrder = self.db.merchantOrder.find_one({"merchantOrderId":merchantOrderId})
                self.parseOrderRawHtml(merchantOrder['orderId'], merchantOrder['subTagId'], merchantOrder['userId'], rawHtml, url, False)['result']
                return 'PARSED_SUCCESS'
                pass
            return 'PARSED_SUCCESS_NO_ORDERS'
        except:
            traceback.print_exc()    
            return 'PARSED_FAILED'
            
    def _getStatusFromDetailedStatus(self, detailedStatus):
        if "ordered from" in detailedStatus.lower():
            return MStore.ORDER_PLACED 

        for key, value in self.orderStatusRegexMap.iteritems():
            if detailedStatus.lower() in value:
                return key
        
        print "Detailed Status need to be mapped"
        print "Found new order status", detailedStatus
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)


def main():
    store = getStore(1)
        
    #store.parseOrderRawHtml(1, 'saad', '2123221', readSSh('/tmp/User8/2015-03-02 12:51:59.545557'), 'https://www.amazon.in/gp/css/summary/edit.html?orderID=12212')
    store.trackOrdersForUser(8,'https://www.amazon.in/gp/css/order-history', readSSh('/tmp/User2/2015-03-03 00:29:40.165513'))

def readSSh(fileName):
    try:
            str1 = open(fileName).read()
            return str1
    except:
        ssh_client = SSHClient()
        str1 = ""
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh_client.connect('104.200.25.40', 22, 'root', 'ecip$dtrMay2014')
        sftp_client = ssh_client.open_sftp()
        try:
            if not os.path.exists(os.path.dirname(fileName)):
                os.makedirs(os.path.dirname(fileName))
            sftp_client.get(fileName, fileName)
            try:
                str1 = open(fileName).read()
                return str1
            finally:
                pass
        except:
            "could not read"
        return str1


if __name__ == '__main__':
        main()
Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/amazon.py – Rev 14447
