Subversion Repositories SmartDukaan

# coding=utf-8

'''

Created on Jan 15, 2015

@author: amit

'''

from base64 import encode

from bs4 import BeautifulSoup

from datetime import datetime

from dtr.dao import Order, SubOrder

from dtr.main import getStore, Store as MStore, ParseException, getBrowserObject, \

    ungzipResponse, tprint

from dtr.sources.flipkart import todict

from paramiko import sftp

from paramiko.client import SSHClient

from paramiko.sftp_client import SFTPClient

import mechanize

import os.path

import paramiko

import re

import time

import traceback

ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'

ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'

THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'

AMAZON_AFF_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/listReports'

class Store(MStore):

    orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],

                            MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],

                            MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],

                            MStore.ORDER_DELIVERED : ['delivered']

                           }

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def getName(self):

        return "flipkart"

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl, insert=False):

        resp = {}

        if ORDER_SUCCESS_URL in orderSuccessUrl or THANKYOU_URL in orderSuccessUrl:

            try:

                soup = BeautifulSoup(rawHtml)

                try:

                    orderUrl = soup.find('div', {"id":"thank-you-box-wrapper"}).div.findAll('div', recursive=False)[1].a['href']

                    merchantOrderId = re.findall(r'.*&oid=(.*)&?.*?', orderUrl)[0]

                except:

                    merchantOrderId = soup.find(id="orders-list").div.span.b.text

                order = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl, False)

                order.orderSuccessUrl = ORDER_REDIRECT_URL % (merchantOrderId)

                order.merchantOrderId = merchantOrderId

                order.requireDetail = True

                order.closed = None

                if self._saveToOrder(todict(order)):

                    resp['result'] = 'ORDER_CREATED'

                    resp["url"] = ORDER_REDIRECT_URL % (merchantOrderId)

                    resp["htmlRequired"] = True

                    resp['orderId'] = orderId

                else:

                    resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

                return resp

            except:

                resp["result"] = 'ORDER_NOT_CREATED'

                return resp

        else:

            try:

                merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

                soup = BeautifulSoup(rawHtml)

                try:

                    self.parseOldStlye(merchantOrder, soup)

                except:

                    try:

                        traceback.print_exc()

                        self.parseNewStlye(merchantOrder, soup)

                    except:

                        traceback.print_exc()

                        self.parseCancelled(merchantOrder, soup)

                resp['result'] = 'ORDER_CREATED'

                return resp    

            except:

                print "Error occurred"

                traceback.print_exc()

                resp['result'] = 'ORDER_NOT_CREATED'

                return resp    

    #This should be exposed from api for specific sources

    def scrapeStoreOrders(self):

        br = getBrowserObject()

        orders = self.db.merchantOrder.find({"storeId":1, "closed":False, "subOrders.closed":False, "subOrders.trackingUrl":{"$exists":True}})

        for merchantOrder in orders:

            try:

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                closed = True

                map1 = {}

                for subOrder in merchantOrder.get("subOrders"):

                    if subOrder.get("closed"):

                        continue

                    elif subOrder.get("trackingUrl") is None:

                        closed = False

                        continue

                    findMap = {"orderId":merchantOrder.get("orderId"), "subOrders.merchantSubOrderId":subOrder.get("merchantSubOrderId")} 

                    trackingUrl = subOrder.get("trackingUrl")

                    if not map1.has_key(trackingUrl):

                        map1[trackingUrl] = self.parseTrackingUrl(br, trackingUrl)

                    newOrder = map1.get(trackingUrl)

                    newOrder['cashBackStatus'] = subOrder.get('cashBackStatus')

                    updateMap = self.getUpdateMap(newOrder)

                    print findMap, "\n", updateMap

                    bulk.find(findMap).update({'$set' : updateMap})

                    closed = closed and newOrder['closed']

                if closed:

                    print "Closed", {"orderId":merchantOrder.get("orderId")},  {"closed":True}

                    bulk.find({"orderId":merchantOrder.get("orderId")}).update({"$set":{"closed":True}})

                result = bulk.execute()

                tprint(result)

            except:

                tprint("Could not update " + str(merchantOrder['orderId']))

                traceback.print_exc()

    def parseOldStlye(self, merchantOrder, soup):

        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl

        table = soup.body.findAll("table", recursive=False)[1]

        #print table

        tables = table.tr.td.findAll("table", recursive=False)

        for tr in tables[2].findAll("tr"):

            boldElement = tr.td.b

            if "Order Placed" in str(boldElement):

                merchantOrder.placedOn = boldElement.next_sibling.strip()

            if "order number" in str(boldElement):

                merchantOrder.merchantOrderId = boldElement.next_sibling.strip()

            if "Order Total" in str(boldElement):

                merchantOrder.paidAmount = int(float(boldElement.find('span').contents[-1].replace(',','')))

        anchors = table.tr.td.findAll("a", recursive=False)

        paymentAnchor = anchors.pop(-1)

        count = 0

        subOrders = []

        merchantOrder.subOrders = subOrders

        counter = 0 

        for anchor in anchors:

            count += 1

            tab = anchor.next_sibling

            status = MStore.ORDER_PLACED

            subStr = "Delivery #" + str(count) + ":"

            if subStr in  tab.find("b").text:

                detailedStatus = tab.find("b").text.replace(subStr, '').strip()

            tab = tab.next_sibling.next_sibling

            trs = tab.find("table").find('tbody').findAll("tr", recursive = False)

            estimatedDelivery = trs[0].td.find("b").next_sibling.strip()

            orderItemTrs = trs[1].findAll("td", recursive=False)[1].table.tbody.findAll("tr", recursive = False)

            i = -1

            for orderItemTr in orderItemTrs:

                i += 1

                if i%2 == 0:

                    continue

                counter += 1

                quantity =  int(re.findall(r'\d+', orderItemTr.td.contents[0])[0])

                productUrl = orderItemTr.td.contents[1].a["href"]

                productTitle = orderItemTr.td.contents[1].a.text

                unitPrice = int(float(orderItemTr.findAll('td')[1].span.text.replace('Rs. ','').replace(',','')))

                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, unitPrice*quantity, status, quantity)

                subOrder.merchantSubOrderId = str(counter) + " of " + merchantOrder.merchantOrderId

                subOrder.estimatedDeliveryDate = estimatedDelivery

                subOrder.productCode = productUrl.split('/')[5]

                subOrder.detailedStatus = detailedStatus

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, unitPrice*quantity)

                cashbackStatus = Store.CB_PENDING

                if cashbackAmount <= 0:

                    cashbackStatus = Store.CB_NA

                subOrder.cashBackStatus = cashbackStatus

                subOrder.cashBackAmount = cashbackAmount

                if percentage > 0:

                    subOrder.cashBackPercentage = percentage

                subOrders.append(subOrder)

        priceList = paymentAnchor.next_sibling.next_sibling.next_sibling.table.table.tbody.tbody.tbody.findAll('tr', recursive=False)

        totalAmount = 0

        grandAmount = 0

        for price in priceList:

            labelTd = price.td

            if 'Subtotal:' in labelTd.text:

                totalAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))

            elif 'Grand Total:' in labelTd.text:

                grandAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))

        if grandAmount < totalAmount:

            diff = totalAmount - grandAmount

            for subOrder in merchantOrder.subOrders:

                subOrder.amountPaid -= int(diff*(1-subOrder.amountPaid/totalAmount))

        self._updateToOrder(todict(merchantOrder))

    def parseNewStlye(self, merchantOrder, soup):

        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl

        orderDetailsContainer = soup.body.find(id="orderDetails")

        divAfterH1 = orderDetailsContainer.h1.next_sibling.next_sibling

        orderLeftDiv = divAfterH1.div

        placedOnSpan = orderLeftDiv.find("span", {'class':'order-date-invoice-item'})

        merchantOrder.placedOn =placedOnSpan.text.split('Ordered on')[1].strip()

        merchantOrder.merchantOrderId = placedOnSpan.next_sibling.next_sibling.text.split('Order#')[1].strip()

        priceBox = divAfterH1.next_sibling.next_sibling.next_sibling.next_sibling.find("div", {"class":"a-box-inner"}).div.div.findAll('div', recursive=False)[-1]

        priceRows = priceBox.findAll('div', {'class':'a-row'})

        subTotal = 0

        shippingPrice = 0   

        promoApplied = 0

        for priceRow in priceRows:

            if "Item(s) Subtotal:" in str(priceRow):

                subTotal = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Shipping:" in str(priceRow):

                shippingPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Grand Total:" in str(priceRow):

                grandPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

                merchantOrder.paidAmount = grandPrice

            elif "Total:" in str(priceRow):

                totalPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Promotion Applied:" in str(priceRow):

                promoApplied  += int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

        totalPaid = subTotal        

        if promoApplied > 0:

            totalPaid -= promoApplied

            if shippingPrice <= promoApplied:

                totalPaid  += shippingPrice

        shipmentDivs = orderDetailsContainer.find('div', {'class':'a-box shipment'}).findAll('div', recursive = False)

        subOrders = []

        merchantOrder.subOrders = subOrders

        i=1

        for shipmentDiv in shipmentDivs:

            innerBoxes = shipmentDiv.findAll('div', recursive = False)

            statusDiv = innerBoxes[0]

            subOrderStatus = statusDiv.div.span.text.strip()

            deliverySpan = statusDiv.div.div.find_next_sibling('div').span

            productDivs = innerBoxes[-1].div.div.div.findAll('div', recursive=False)

            subOrders = []

            merchantOrder.subOrders = subOrders

            for i, productDiv in enumerate(productDivs):

                i +=1 

                imgDiv  = productDiv.div.div

                detailDiv = imgDiv.find_next_sibling('div')

                detailDivs = detailDiv.findAll('div', recursive=False)

                arr = detailDivs[0].a.text.strip().split(" of ", 1)

                (productTitle, quantity) = (arr[-1], (1 if len(arr)==1 else int(arr[0])) )

                unitPrice = int(float(detailDivs[2].span.text.replace('Rs. ','').replace(',','')))

                amountPaid = int((unitPrice*quantity*totalPaid)/subTotal)

                productUrl = "http://www.amazon.in" + detailDivs[0].a.get('href')

                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid, MStore.ORDER_PLACED, quantity)

                subOrder.productCode = productUrl.split('/')[5]

                subOrder.unitPrice = unitPrice

                subOrder.merchantSubOrderId = str(i) + " of " + merchantOrder.merchantOrderId

                if deliverySpan is not None:

                    subOrder.estimatedDeliveryDate = deliverySpan.span.text.strip() 

                subOrder.detailedStatus = subOrderStatus

                subOrder.deliveryCharges = shippingPrice

                subOrder.imgUrl = imgDiv.img["src"]

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amountPaid)

                cashbackStatus = Store.CB_PENDING

                if cashbackAmount <= 0:

                    cashbackStatus = Store.CB_NA

                subOrder.cashBackStatus = cashbackStatus

                subOrder.cashBackAmount = cashbackAmount

                if percentage > 0:

                    subOrder.cashBackPercentage = percentage

                subOrders.append(subOrder)

        self._updateToOrder(todict(merchantOrder))

    def parseCancelled(self, merchantOrder,soup):

        fonts = soup.body.findAll("table", recursive=False)[1].findAll("font")

        if fonts[0].text == "Important Message":

            if fonts[1].text=="This order has been cancelled.":

                merchantOrder.closed = True

                merchantOrder.status = "Cancelled"

                merchantOrder.requireDetail = False

                self._updateToOrder(todict(merchantOrder))

            else:

                raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

        else:

            orderDetails = soup.body.find(id="orderDetails")

            if orderDetails is not None and orderDetails.h4.text == "This order has been cancelled.":

                merchantOrder.closed = True

                merchantOrder.status = "Cancelled"

                merchantOrder.requireDetail = False

                self._updateToOrder(todict(merchantOrder))

            else:

                raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

    def getTrackingUrls(self, userId):

        missingOrderUrls = []

        missingOrders = self._getMissingOrders({'userId':userId})

        for missingOrder in missingOrders:

            missingOrderUrls.append(ORDER_REDIRECT_URL%(missingOrder['merchantOrderId']))

        orders = self._getActiveOrders({'userId':userId, "$exists":{"subOrders.trackingUrl":False} })

        count = len(orders)

        print "count", count

        if count > 0:

            return missingOrderUrls + ['https://www.amazon.in/gp/css/order-history', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled', 'https://www.amazon.in/gp/css/order-history?orderFilter=cancelled&startIndex=10']

        else: 

            return missingOrderUrls

    def trackOrdersForUser(self, userId, url, rawHtml):

        directory = "/tmp/User" + str(userId)

        if not os.path.exists(directory):

            os.makedirs(directory)

        filename = directory + "/" + str(datetime.now())

        print "filename---", filename

        f = open(filename,'w')

        f.write(rawHtml) # python will convert \n to os.linesep

        f.close() # you can omit in most cases as the destructor will call if

        try:

            searchMap = {'userId':userId}

            collectionMap = {'merchantOrderId':1}

            activeOrders = self._getActiveOrders(searchMap, collectionMap)

            datetimeNow = datetime.now()

            timestamp = int(time.mktime(datetimeNow.timetuple()))

            print "url----------------", url

            if url == 'https://www.amazon.in/gp/css/order-history' or 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled' in url:

                soup = BeautifulSoup(rawHtml)

                allOrders = soup.find(id="ordersContainer").findAll('div', {'class':'a-box-group a-spacing-base order'})

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                for activeOrder in activeOrders:

                    for orderEle in allOrders:

                        orderdiv = orderEle.find('div', {'class':'a-box a-color-offset-background order-info'}).find('div', {'class':'a-fixed-right-grid-col actions a-col-right'})

                        merchantOrderId = orderdiv.find('span', {'class':'a-color-secondary value'}).text.strip()

                        if merchantOrderId==activeOrder['merchantOrderId']:

                            closed = True

                            shipments = orderEle.findAll('div',{'class':re.compile('.*?a-box.*?')}, recursive=False)

                            shipments.pop(0)

                            for shipment in shipments:

                                shipdiv = shipment.find('div', {'class':'a-box-inner'})

                                sdivs = shipment.div.div.findAll('div', recursive=False)

                                orderStatus = sdivs[0].span.text.strip()

                                status = self._getStatusFromDetailedStatus(orderStatus)

                                if status == MStore.ORDER_DELIVERED:

                                    deliveredOn = sdivs[0].findAll('span')[-1].text.strip()

                                    deliveredOn = deliveredOn.split(":")[1].strip()

                                deliveryestimatespan = sdivs[0].find('span', {'class':'a-color-success'})

                                deliveryEstimate = None

                                if deliveryestimatespan is not None:

                                    deliveryEstimate = deliveryestimatespan.find('span', {'class':'a-text-bold'}).text.strip()

                                productDivs = shipdiv.find('div', {'class':re.compile('.*?a-spacing-top-medium.*?')}).find('div', {'class':'a-row'}).findAll('div', recursive=False)

                                trackingUrl = None

                                for buttonDiv in shipdiv.findAll('span', {'class':'a-button-inner'}):

                                    if buttonDiv.find('a').text.strip()=='Track package':

                                        trackingUrl = buttonDiv.find('a')['href'].strip()

                                        if not trackingUrl.startswith("http"):

                                            trackingUrl = "http://www.amazon.in" + trackingUrl

                                        break

                                for prodDiv in productDivs:

                                    prodDiv.find('div', {'class':'a-fixed-left-grid-inner'})

                                    productTitle = prodDiv.find('div', {'class':'a-fixed-left-grid-inner'}).find("div", {'class':'a-row'}).find('a').text.strip()

                                    imgUrl = prodDiv.find("img")["src"]

                                    for subOrder in activeOrder['subOrders']:

                                        if subOrder['productTitle'] == productTitle:

                                            findMap = {"orderId": activeOrder['orderId'], "subOrders.merchantSubOrderId": subOrder.get("merchantSubOrderId")}

                                            updateMap = {}

                                            closedStatus = False

                                            updateMap['subOrders.$.imgUrl'] = imgUrl

                                            updateMap['subOrders.$.lastTracked'] = timestamp

                                            updateMap['subOrders.$.detailedStatus'] = orderStatus

                                            cashbackStatus = subOrder.get("cashBackStatus")

                                            updateMap['subOrders.$.status'] = status 

                                            if status==MStore.ORDER_DELIVERED:                               

                                                updateMap['subOrders.$.deliveredOn'] = deliveredOn

                                                closedStatus = True

                                                updateMap['subOrders.$.closed'] = True

                                                if cashbackStatus == Store.CB_PENDING:

                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_APPROVED

                                            if status==MStore.ORDER_CANCELLED:     

                                                closedStatus = True

                                                updateMap['subOrders.$.closed'] = True

                                                if cashbackStatus == Store.CB_PENDING:

                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_CANCELLED

                                            if status==MStore.ORDER_SHIPPED:   

                                                updateMap['subOrders.$.estimatedDeliveryDate'] = deliveryEstimate

                                                if trackingUrl is not None:

                                                    updateMap['subOrders.$.trackingUrl'] = trackingUrl

                                            if not closedStatus:

                                                closed = False

                                            bulk.find(findMap).update({'$set' : updateMap})

                                            break

                            bulk.find({'orderId': activeOrder['orderId']}).update({'$set':{'closed': closed}})            

                bulk.execute()

                return 'PARSED_SUCCESS'

            else:

                merchantOrderId = re.findall(r'https://www.amazon.in/gp/css/summary/edit.html\?orderID=(.*)?', url, re.IGNORECASE)[0] 

                merchantOrder = self.db.merchantOrder.find_one({"merchantOrderId":merchantOrderId})

                self.parseOrderRawHtml(merchantOrder['orderId'], merchantOrder['subTagId'], merchantOrder['userId'], rawHtml, url, False)['result']

                return 'PARSED_SUCCESS'

                pass

            return 'PARSED_SUCCESS_NO_ORDERS'

        except:

            traceback.print_exc()    

            return 'PARSED_FAILED'

    def _getStatusFromDetailedStatus(self, detailedStatus):

        if "ordered from" in detailedStatus.lower():

            return MStore.ORDER_PLACED 

        for key, value in self.orderStatusRegexMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped"

        print "Found new order status", detailedStatus

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)

    def scrapeAffiliate(self, startDate=None, endDate=None):

        pass

    def parseTrackingUrl(self, br, trackingUrl):

        subOrder = {}

        response = br.open(trackingUrl)

        page = ungzipResponse(response)

        soup = BeautifulSoup(page)

        alertContainer = soup.find("div", {"class":"a-box-inner a-alert-container"})

        if alertContainer is not None:

            statusText = alertContainer.h4.span.text

            detailedStatus = statusText.split(":")[0].strip()

            subOrder['detailedStatus'] = detailedStatus

            if detailedStatus.lower() == "in transit":

                pass

            elif detailedStatus.lower() == "undeliverable":

                subOrder['status'] = MStore.ORDER_CANCELLED

                return subOrder

        else:

            summaryLeft = soup.find(id="summaryLeft")

            statusText = summaryLeft.h2.text

            detailedStatus = statusText.split(":")[0].strip()

            subOrder['detailedStatus'] = detailedStatus

            if detailedStatus.lower() == "delivered":

                subOrder['deliveredOn'] = summaryLeft.findAll("span")[-2].text.strip()

                subOrder['status'] = MStore.ORDER_DELIVERED

                return subOrder

            else:

                subOrder['expectedDelivery'] = summaryLeft.findAll("span")[-1].text.strip()

        return subOrder

def main():

    store = getStore(1)

    br = mechanize.Browser()

    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),

                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),

                     ('Accept-Encoding', 'gzip,deflate,sdch'),                  

                     ('Accept-Language', 'en-US,en;q=0.8'),                     

                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]

    store.scrapeStoreOrders()

    #store.parseOrderRawHtml(2121, "13232", 14, readSSh("/home/amit/917.html"), "https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html/ref=ox_spc_place_order?ie=UTF8&hasWorkingJavascript=")

def readSSh(fileName):

    try:

            str1 = open(fileName).read()

            return str1

    except:

        ssh_client = SSHClient()

        str1 = ""

        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        ssh_client.connect('104.200.25.40', 22, 'root', 'ecip$dtrMay2014')

        sftp_client = ssh_client.open_sftp()

        try:

            if not os.path.exists(os.path.dirname(fileName)):

                os.makedirs(os.path.dirname(fileName))

            sftp_client.get(fileName, fileName)

            try:

                str1 = open(fileName).read()

                return str1

            finally:

                pass

        except:

            "could not read"

        return str1

if __name__ == '__main__':

        main()