Subversion Repositories SmartDukaan

# coding=utf-8

'''

Created on Jan 15, 2015

@author: amit

'''

from base64 import encode

from bs4 import BeautifulSoup

from datetime import datetime

from dtr.dao import Order, SubOrder

from dtr.main import getStore, Store as MStore, ParseException

from dtr.sources.flipkart import todict

from paramiko import sftp

from paramiko.client import SSHClient

from paramiko.sftp_client import SFTPClient

import os.path

import paramiko

import re

import time

import traceback

ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'

ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'

THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'

class Store(MStore):

    orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],

                            MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],

                            MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],

                            MStore.ORDER_DELIVERED : ['delivered']

                           }

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def getName(self):

        return "flipkart"

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl, insert=False):

        resp = {}

        if ORDER_SUCCESS_URL in orderSuccessUrl or THANKYOU_URL in orderSuccessUrl:

            try:

                soup = BeautifulSoup(rawHtml)

                orderUrl = soup.find('div', {"id":"thank-you-box-wrapper"}).div.findAll('div', recursive=False)[1].a['href']

                merchantOrderId = re.findall(r'.*&oid=(.*)&?.*?', orderUrl)[0]

                order = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl, False)

                order.orderSuccessUrl = ORDER_REDIRECT_URL % (merchantOrderId)

                order.merchantOrderId = merchantOrderId

                order.requireDetail = True

                order.closed = None

                if self._saveToOrder(todict(order)):

                    resp['result'] = 'ORDER_CREATED'

                    resp["url"] = ORDER_REDIRECT_URL % (merchantOrderId)

                    resp["htmlRequired"] = True

                    resp['orderId'] = orderId

                else:

                    resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

                return resp

            except:

                resp["result"] = 'ORDER_NOT_CREATED'

                return resp

        else:

            try:

                merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

                merchantOrder.orderTrackingUrl = orderSuccessUrl

                soup = BeautifulSoup(rawHtml,from_encoding="utf-8")

                table = soup.body.findAll("table", recursive=False)[1]

                #print table

                tables = table.tr.td.findAll("table", recursive=False)

                for tr in tables[2].findAll("tr"):

                    boldElement = tr.td.b

                    if "Order Placed" in str(boldElement):

                        merchantOrder.placedOn = boldElement.next_sibling.strip()

                    if "order number" in str(boldElement):

                        merchantOrder.merchantOrderId = boldElement.next_sibling.strip()

                    if "Order Total" in str(boldElement):

                        merchantOrder.paidAmount = int(float(boldElement.find('span').contents[-1].replace(',','')))

                anchors = table.tr.td.findAll("a", recursive=False)

                paymentAnchor = anchors.pop(-1)

                count = 0

                subOrders = []

                merchantOrder.subOrders = subOrders

                for anchor in anchors:

                    count += 1

                    tab = anchor.next_sibling

                    status = MStore.ORDER_PLACED

                    subStr = "Delivery #" + str(count) + ":"

                    if subStr in  tab.find("b").text:

                        detailedStatus = tab.find("b").text.replace(subStr, '').strip()

                    tab = tab.next_sibling.next_sibling

                    trs = tab.find("table").find('tbody').findAll("tr", recursive = False)

                    estimatedDelivery = trs[0].td.find("b").next_sibling.strip()

                    orderItemTrs = trs[1].findAll("td", recursive=False)[1].table.tbody.findAll("tr", recursive = False)

                    i = -1

                    for orderItemTr in orderItemTrs:

                        i += 1

                        if i%2 == 0:

                            continue

                        quantity =  int(re.findall(r'\d+', orderItemTr.td.contents[0])[0])

                        productUrl = orderItemTr.td.contents[1].a["href"]

                        productTitle = orderItemTr.td.contents[1].a.text

                        unitPrice = int(float(orderItemTr.findAll('td')[1].span.text.replace('Rs. ','').replace(',','')))

                        subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, unitPrice*quantity, status, quantity)

                        subOrder.merchantSubOrderId = orderItemTr.find("input")["name"]

                        subOrder.estimatedDeliveryDate = estimatedDelivery

                        subOrder.productCode = productUrl.split('/')[5]

                        subOrder.detailedStatus = detailedStatus

                        (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, unitPrice*quantity)

                        cashbackStatus = Store.CB_PENDING

                        if cashbackAmount <= 0:

                            cashbackStatus = Store.CB_NA

                        subOrder.cashBackStatus = cashbackStatus

                        subOrder.cashBackAmount = cashbackAmount

                        if percentage > 0:

                            subOrder.cashBackPercentage = percentage

                        subOrders.append(subOrder)

                priceList = paymentAnchor.next_sibling.next_sibling.next_sibling.table.table.tbody.tbody.tbody.findAll('tr', recursive=False)

                totalAmount = 0

                grandAmount = 0

                for price in priceList:

                    labelTd = price.td

                    if 'Subtotal:' in labelTd.text:

                        totalAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))

                    elif 'Grand Total:' in labelTd.text:

                        grandAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))

                if grandAmount < totalAmount:

                    diff = totalAmount - grandAmount

                    for subOrder in merchantOrder.subOrders:

                        subOrder.amountPaid -= int(diff*(1-subOrder.amountPaid/totalAmount))

                self._updateToOrder(todict(merchantOrder))

                resp['result'] = 'ORDER_CREATED'

                return resp    

            except:

                print "Error occurred"

                traceback.print_exc()

                resp['result'] = 'ORDER_NOT_CREATED'

                return resp    

    #This should be exposed from api for specific sources

    def scrapeStoreOrders(self):

        pass

    def getTrackingUrls(self, userId):

        missingOrderUrls = []

        missingOrders = self._getMissingOrders({'userId':userId})

        for missingOrder in missingOrders:

            missingOrderUrls.append(ORDER_REDIRECT_URL%(missingOrder['merchantOrderId']))

        orders = self._getActiveOrders({'userId':userId})

        count = len(orders)

        print "count", count

        if count > 0:

            return missingOrderUrls + ['https://www.amazon.in/gp/css/order-history', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled', 'https://www.amazon.in/gp/css/order-history?orderFilter=cancelled&startIndex=10']

        else: 

            return missingOrderUrls

    def trackOrdersForUser(self, userId, url, rawHtml):

        directory = "/tmp/User" + str(userId)

        if not os.path.exists(directory):

            os.makedirs(directory)

        filename = directory + "/" + str(datetime.now())

        print "filename---", filename

        f = open(filename,'w')

        f.write(rawHtml) # python will convert \n to os.linesep

        f.close() # you can omit in most cases as the destructor will call if

        try:

            searchMap = {'userId':userId}

            collectionMap = {'merchantOrderId':1}

            activeOrders = self._getActiveOrders(searchMap, collectionMap)

            datetimeNow = datetime.now()

            timestamp = int(time.mktime(datetimeNow.timetuple()))

            print "url----------------", url

            if url == 'https://www.amazon.in/gp/css/order-history' or 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled' in url:

                soup = BeautifulSoup(rawHtml)

                allOrders = soup.find(id="ordersContainer").findAll('div', {'class':'a-box-group a-spacing-base order'})

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                for activeOrder in activeOrders:

                    for orderEle in allOrders:

                        orderdiv = orderEle.find('div', {'class':'a-box a-color-offset-background order-info'}).find('div', {'class':'a-fixed-right-grid-col actions a-col-right'})

                        merchantOrderId = orderdiv.find('span', {'class':'a-color-secondary value'}).text.strip()

                        if merchantOrderId==activeOrder['merchantOrderId']:

                            closed = True

                            shipments = orderEle.findAll('div',{'class':re.compile('.*?a-box.*?')}, recursive=False)

                            shipments.pop(0)

                            for shipment in shipments:

                                shipdiv = shipment.find('div', {'class':'a-box-inner'})

                                sdivs = shipment.div.div.findAll('div', recursive=False)

                                orderStatus = sdivs[0].span.text.strip()

                                status = self._getStatusFromDetailedStatus(orderStatus)

                                if status == MStore.ORDER_DELIVERED:

                                    deliveredOn = sdivs[0].findAll('span')[-1].text.strip()

                                    deliveredOn = deliveredOn.split(":")[1].strip()

                                else:

                                    deliveryestimatespan = sdivs[0].find('span', {'class':'a-color-success'})

                                    deliveryEstimate = None

                                    if deliveryestimatespan is not None:

                                        deliveryEstimate = deliveryestimatespan.find('span', {'class':'a-text-bold'}).text.strip()

                                    productDivs = shipdiv.find('div', {'class':re.compile('.*?a-spacing-top-medium.*?')}).find('div', {'class':'a-row'}).findAll('div', recursive=False)

                                trackingUrl = None

                                for buttonDiv in shipdiv.findAll('span', {'class':'a-button-inner'}):

                                    if buttonDiv.find('a').text.strip()=='Track package':

                                        trackingUrl = buttonDiv.find('a')['href'].strip()

                                        if not trackingUrl.startswith("http"):

                                            trackingUrl = "https://www.amazon.in/" + trackingUrl

                                        break

                                for prodDiv in productDivs:

                                    prodDiv.find('div', {'class':'a-fixed-left-grid-inner'})

                                    productTitle = prodDiv.find('div', {'class':'a-fixed-left-grid-inner'}).find("div", {'class':'a-row'}).find('a').text.strip()

                                    imgUrl = prodDiv.find("img")["src"]

                                    for subOrder in activeOrder['subOrders']:

                                        if subOrder['productTitle'] == productTitle:

                                            findMap = {"orderId": activeOrder['orderId'], "subOrders.merchantSubOrderId": subOrder.get("merchantSubOrderId")}

                                            updateMap = {}

                                            closedStatus = False

                                            updateMap['subOrders.$.imgUrl'] = imgUrl

                                            updateMap['subOrders.$.lastTracked'] = timestamp

                                            updateMap['subOrders.$.detailedStatus'] = orderStatus

                                            cashbackStatus = subOrder.get("cashBackStatus")

                                            updateMap['subOrders.$.status'] = status 

                                            if status==MStore.ORDER_DELIVERED:                               

                                                updateMap['subOrders.$.deliveredOn'] = deliveredOn

                                                closedStatus = True

                                                updateMap['subOrders.$.closed'] = True

                                                if cashbackStatus == Store.CB_PENDING:

                                                    updateMap['subOrders.$.cashbackStatus'] = Store.CB_APPROVED

                                            if status==MStore.ORDER_CANCELLED:     

                                                closedStatus = True

                                                updateMap['subOrders.$.closed'] = True

                                                if cashbackStatus == Store.CB_PENDING:

                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_CANCELLED

                                            if status==MStore.ORDER_SHIPPED:   

                                                updateMap['subOrders.$.estimatedDeliveryDate'] = deliveryEstimate

                                                if trackingUrl is not None:

                                                    updateMap['subOrders.$.trackingUrl'] = trackingUrl

                                            if not closedStatus:

                                                closed = False

                                            bulk.find(findMap).update({'$set' : updateMap})

                                            break

                            bulk.find({'orderId': activeOrder['orderId']}).update({'$set':{'closed': closed}})            

                bulk.execute()

                return 'PARSED_SUCCESS'

            else:

                merchantOrderId = re.findall(r'https://www.amazon.in/gp/css/summary/edit.html\?orderID=(.*)?', url, re.IGNORECASE)[0] 

                merchantOrder = self.db.merchantOrder.find_one({"merchantOrderId":merchantOrderId})

                self.parseOrderRawHtml(merchantOrder['orderId'], merchantOrder['subTagId'], merchantOrder['userId'], rawHtml, url, False)['result']

                return 'PARSED_SUCCESS'

                pass

            return 'PARSED_SUCCESS_NO_ORDERS'

        except:

            traceback.print_exc()    

            return 'PARSED_FAILED'

    def _getStatusFromDetailedStatus(self, detailedStatus):

        if "ordered from" in detailedStatus.lower():

            return MStore.ORDER_PLACED 

        for key, value in self.orderStatusRegexMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped"

        print "Found new order status", detailedStatus

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)

def main():

    store = getStore(1)

    #store.parseOrderRawHtml(1, 'saad', '2123221', readSSh('/tmp/User8/2015-03-02 12:51:59.545557'), 'https://www.amazon.in/gp/css/summary/edit.html?orderID=12212')

    store.trackOrdersForUser(8,'https://www.amazon.in/gp/css/order-history', readSSh('/tmp/User2/2015-03-02 23:02:16.531611'))

def readSSh(fileName):

    try:

            str1 = open(fileName).read()

            return str1

    except:

        ssh_client = SSHClient()

        str1 = ""

        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        ssh_client.connect('104.200.25.40', 22, 'root', 'ecip$dtrMay2014')

        sftp_client = ssh_client.open_sftp()

        try:

            if not os.path.exists(os.path.dirname(fileName)):

                os.makedirs(os.path.dirname(fileName))

            sftp_client.get(fileName, fileName)

            try:

                str1 = open(fileName).read()

                return str1

            finally:

                pass

        except:

            "could not read"

        return str1

if __name__ == '__main__':

        main()