Subversion Repositories SmartDukaan

# coding=utf-8
'''
Created on Jan 15, 2015

@author: amit
'''
from base64 import encode
from bs4 import BeautifulSoup
from datetime import datetime
from dtr.dao import Order, SubOrder
from dtr.main import getStore, Store as MStore, ParseException, getBrowserObject, \
    ungzipResponse
from dtr.sources.flipkart import todict
from paramiko import sftp
from paramiko.client import SSHClient
from paramiko.sftp_client import SFTPClient
import os.path
import paramiko
import re
import time
import traceback

ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'
ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'
THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'
class Store(MStore):
    
    orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],
                            MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],
                            MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],
                            MStore.ORDER_DELIVERED : ['delivered']
                           }

    def __init__(self,store_id):
        super(Store, self).__init__(store_id)
        
    def getName(self):
        return "flipkart"
    
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl, insert=False):
        resp = {}
        if ORDER_SUCCESS_URL in orderSuccessUrl or THANKYOU_URL in orderSuccessUrl:
            try:
                soup = BeautifulSoup(rawHtml)
                orderUrl = soup.find('div', {"id":"thank-you-box-wrapper"}).div.findAll('div', recursive=False)[1].a['href']
                merchantOrderId = re.findall(r'.*&oid=(.*)&?.*?', orderUrl)[0]
                order = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl, False)
                order.orderSuccessUrl = ORDER_REDIRECT_URL % (merchantOrderId)
                order.merchantOrderId = merchantOrderId
                order.requireDetail = True
                order.closed = None
                if self._saveToOrder(todict(order)):
                    resp['result'] = 'ORDER_CREATED'
                    resp["url"] = ORDER_REDIRECT_URL % (merchantOrderId)
                    resp["htmlRequired"] = True
                    resp['orderId'] = orderId
                else:
                    resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                    
                return resp
            except:
                resp["result"] = 'ORDER_NOT_CREATED'
                return resp
        
        else:
            try:
                merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
                soup = BeautifulSoup(rawHtml)
                try:
                    self.parseOldStlye(merchantOrder, soup)
                except:
                    try:
                        self.parseNewStlye(merchantOrder, soup)
                    except:
                        self.parseCancelled(merchantOrder, soup)
                resp['result'] = 'ORDER_CREATED'
                return resp    
            except:
                print "Error occurred"
                traceback.print_exc()
                resp['result'] = 'ORDER_NOT_CREATED'
                return resp    
                    
    #This should be exposed from api for specific sources
    def scrapeStoreOrders(self):
        pass
    
    def parseOldStlye(self, merchantOrder, soup):
        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl
        table = soup.body.findAll("table", recursive=False)[1]
        #print table
        tables = table.tr.td.findAll("table", recursive=False)
        for tr in tables[2].findAll("tr"):
            boldElement = tr.td.b
            if "Order Placed" in str(boldElement):
                merchantOrder.placedOn = boldElement.next_sibling.strip()
            if "order number" in str(boldElement):
                merchantOrder.merchantOrderId = boldElement.next_sibling.strip()
            if "Order Total" in str(boldElement):
                merchantOrder.paidAmount = int(float(boldElement.find('span').contents[-1].replace(',','')))
        anchors = table.tr.td.findAll("a", recursive=False)
        paymentAnchor = anchors.pop(-1)
        
        count = 0
        subOrders = []
        merchantOrder.subOrders = subOrders
        counter = 0 
        for anchor in anchors:
            count += 1
            tab = anchor.next_sibling
            status = MStore.ORDER_PLACED
            subStr = "Delivery #" + str(count) + ":"
            if subStr in  tab.find("b").text:
                detailedStatus = tab.find("b").text.replace(subStr, '').strip()
            
            tab = tab.next_sibling.next_sibling
            trs = tab.find("table").find('tbody').findAll("tr", recursive = False)
            
            estimatedDelivery = trs[0].td.find("b").next_sibling.strip()
            
            orderItemTrs = trs[1].findAll("td", recursive=False)[1].table.tbody.findAll("tr", recursive = False)
            i = -1
            for orderItemTr in orderItemTrs:
                i += 1
                if i%2 == 0:
                    continue
                counter += 1
                quantity =  int(re.findall(r'\d+', orderItemTr.td.contents[0])[0])
                
                productUrl = orderItemTr.td.contents[1].a["href"]
                productTitle = orderItemTr.td.contents[1].a.text
                
                unitPrice = int(float(orderItemTr.findAll('td')[1].span.text.replace('Rs. ','').replace(',','')))
                
            
                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, unitPrice*quantity, status, quantity)
                subOrder.merchantSubOrderId = str(counter) + " of " + merchantOrder.merchantOrderId
                subOrder.estimatedDeliveryDate = estimatedDelivery
                subOrder.productCode = productUrl.split('/')[5]
                subOrder.detailedStatus = detailedStatus
                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, unitPrice*quantity)
                cashbackStatus = Store.CB_PENDING
                if cashbackAmount <= 0:
                    cashbackStatus = Store.CB_NA
                subOrder.cashBackStatus = cashbackStatus
                subOrder.cashBackAmount = cashbackAmount
                if percentage > 0:
                    subOrder.cashBackPercentage = percentage
                subOrders.append(subOrder)
        priceList = paymentAnchor.next_sibling.next_sibling.next_sibling.table.table.tbody.tbody.tbody.findAll('tr', recursive=False)
        totalAmount = 0
        grandAmount = 0
        for price in priceList:
            labelTd = price.td
            if 'Subtotal:' in labelTd.text:
                totalAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))
            elif 'Grand Total:' in labelTd.text:
                grandAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))
        if grandAmount < totalAmount:
            diff = totalAmount - grandAmount
            for subOrder in merchantOrder.subOrders:
                subOrder.amountPaid -= int(diff*(1-subOrder.amountPaid/totalAmount))
        self._updateToOrder(todict(merchantOrder))

    def parseNewStlye(self, merchantOrder, soup):
        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl
        orderDetailsContainer = soup.body.find(id="orderDetails")
        orderLeftDiv = orderDetailsContainer.h1.next_sibling.next_sibling.div
        placedOnSpan = orderLeftDiv.find("span", {'class':'order-date-invoice-item'})
        merchantOrder.placedOn =placedOnSpan.text.split('Ordered on')[1].strip()
        merchantOrder.merchantOrderId = placedOnSpan.next_sibling.next_sibling.text.split('Order#')[1].strip()
        priceBox = orderDetailsContainer.find('div', {'class':re.compile(r'\ba-box-inner\b')}).div.div.findAll('div', recursive=False)[-1]
        priceRows = priceBox.findAll('div', {'class':'a-row'})
        subTotal = 0
        shippingPrice = 0
        promoApplied = 0
        for priceRow in priceRows:
            if "Item(s) Subtotal:" in str(priceRow):
                subTotal = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
            elif "Shipping:" in str(priceRow):
                shippingPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
            elif "Grand Total:" in str(priceRow):
                grandPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
                merchantOrder.paidAmount = grandPrice
            elif "Total:" in str(priceRow):
                totalPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
            elif "Promotion Applied:" in str(priceRow):
                promoApplied  += int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
        totalPaid = subTotal        
        if promoApplied > 0:
            totalPaid -= promoApplied
            if shippingPrice <= promoApplied:
                totalPaid  += shippingPrice

        shipmentDivs = orderDetailsContainer.find('div', {'class':'a-box shipment'}).findAll('div', recursive = False)
        subOrders = []
        merchantOrder.subOrders = subOrders
        i=1
        for shipmentDiv in shipmentDivs:
            innerBoxes = shipmentDiv.findAll('div', recursive = False)
            statusDiv = innerBoxes[0]
            subOrderStatus = statusDiv.div.span.text.strip()
            estimatedDeliveryDate = statusDiv.div.div.find_next_sibling('div').span.span.text.strip()
            productDivs = innerBoxes[-1].div.div.div.findAll('div', recursive=False)
            subOrders = []
            merchantOrder.subOrders = subOrders
            for i, productDiv in enumerate(productDivs):
                i +=1 
                imgDiv  = productDiv.div.div
                detailDiv = imgDiv.find_next_sibling('div')
                detailDivs = detailDiv.findAll('div', recursive=False)
                arr = detailDivs[0].a.text.strip().split(" of ", 1)
                (productTitle, quantity) = (arr[-1], (1 if len(arr)==1 else int(arr[0])) )
                unitPrice = int(float(detailDivs[2].span.text.replace('Rs. ','').replace(',','')))
                amountPaid = int((unitPrice*quantity*totalPaid)/subTotal)
                productUrl = "http://www.amazon.in" + detailDivs[0].a.get('href')
                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid, MStore.ORDER_PLACED, quantity)
                subOrder.productCode = productUrl.split('/')[5]
                subOrder.unitPrice = unitPrice
                subOrder.merchantSubOrderId = str(i) + " of " + merchantOrder.merchantOrderId 
                subOrder.estimatedDeliveryDate = estimatedDeliveryDate
                subOrder.detailedStatus = subOrderStatus
                subOrder.deliveryCharges = shippingPrice
                subOrder.imgUrl = imgDiv.img["src"]
                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amountPaid)
                cashbackStatus = Store.CB_PENDING
                if cashbackAmount <= 0:
                    cashbackStatus = Store.CB_NA
                subOrder.cashBackStatus = cashbackStatus
                subOrder.cashBackAmount = cashbackAmount
                if percentage > 0:
                    subOrder.cashBackPercentage = percentage
                subOrders.append(subOrder)
        self._updateToOrder(todict(merchantOrder))
        
    def parseCancelled(self, merchantOrder,soup):
        fonts = soup.body.findAll("table", recursive=False)[1].findAll("font")
        if fonts[0].text == "Important Message":
            if fonts[1].text=="This order has been cancelled.":
                merchantOrder.closed = True
                merchantOrder.status = "Cancelled"
                merchantOrder.requireDetail = False
                self._updateToOrder(todict(merchantOrder))
            else:
                raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

    def getTrackingUrls(self, userId):
        
        missingOrderUrls = []
        missingOrders = self._getMissingOrders({'userId':userId})
        for missingOrder in missingOrders:
            missingOrderUrls.append(ORDER_REDIRECT_URL%(missingOrder['merchantOrderId']))
        orders = self._getActiveOrders({'userId':userId})
        count = len(orders)
        print "count", count
        if count > 0:
            return missingOrderUrls + ['https://www.amazon.in/gp/css/order-history', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled', 'https://www.amazon.in/gp/css/order-history?orderFilter=cancelled&startIndex=10']
        else: 
            return missingOrderUrls
            
    def trackOrdersForUser(self, userId, url, rawHtml):
        directory = "/tmp/User" + str(userId)
        if not os.path.exists(directory):
            os.makedirs(directory)
        filename = directory + "/" + str(datetime.now())
        print "filename---", filename
        f = open(filename,'w')
        f.write(rawHtml) # python will convert \n to os.linesep
        f.close() # you can omit in most cases as the destructor will call if
        
        try:
            searchMap = {'userId':userId}
            collectionMap = {'merchantOrderId':1}
            activeOrders = self._getActiveOrders(searchMap, collectionMap)
            datetimeNow = datetime.now()
            timestamp = int(time.mktime(datetimeNow.timetuple()))
            print "url----------------", url
            
            if url == 'https://www.amazon.in/gp/css/order-history' or 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled' in url:
                soup = BeautifulSoup(rawHtml)
                allOrders = soup.find(id="ordersContainer").findAll('div', {'class':'a-box-group a-spacing-base order'})
                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
                for activeOrder in activeOrders:
                    for orderEle in allOrders:
                        orderdiv = orderEle.find('div', {'class':'a-box a-color-offset-background order-info'}).find('div', {'class':'a-fixed-right-grid-col actions a-col-right'})
                        merchantOrderId = orderdiv.find('span', {'class':'a-color-secondary value'}).text.strip()
                        if merchantOrderId==activeOrder['merchantOrderId']:
                            closed = True
                            shipments = orderEle.findAll('div',{'class':re.compile('.*?a-box.*?')}, recursive=False)
                            shipments.pop(0)
                            for shipment in shipments:
                                shipdiv = shipment.find('div', {'class':'a-box-inner'})
                                sdivs = shipment.div.div.findAll('div', recursive=False)
                                orderStatus = sdivs[0].span.text.strip()
                                status = self._getStatusFromDetailedStatus(orderStatus)
                                if status == MStore.ORDER_DELIVERED:
                                    deliveredOn = sdivs[0].findAll('span')[-1].text.strip()
                                    deliveredOn = deliveredOn.split(":")[1].strip()
                                deliveryestimatespan = sdivs[0].find('span', {'class':'a-color-success'})
                                deliveryEstimate = None
                                if deliveryestimatespan is not None:
                                    deliveryEstimate = deliveryestimatespan.find('span', {'class':'a-text-bold'}).text.strip()
                                productDivs = shipdiv.find('div', {'class':re.compile('.*?a-spacing-top-medium.*?')}).find('div', {'class':'a-row'}).findAll('div', recursive=False)
                                trackingUrl = None
                                for buttonDiv in shipdiv.findAll('span', {'class':'a-button-inner'}):
                                    if buttonDiv.find('a').text.strip()=='Track package':
                                        trackingUrl = buttonDiv.find('a')['href'].strip()
                                        if not trackingUrl.startswith("http"):
                                            trackingUrl = "https://www.amazon.in/" + trackingUrl
                                        break
                                for prodDiv in productDivs:
                                    prodDiv.find('div', {'class':'a-fixed-left-grid-inner'})
                                    productTitle = prodDiv.find('div', {'class':'a-fixed-left-grid-inner'}).find("div", {'class':'a-row'}).find('a').text.strip()
                                    imgUrl = prodDiv.find("img")["src"]
                                    for subOrder in activeOrder['subOrders']:
                                        if subOrder['productTitle'] == productTitle:
                                            findMap = {"orderId": activeOrder['orderId'], "subOrders.merchantSubOrderId": subOrder.get("merchantSubOrderId")}
                                            updateMap = {}
                                            closedStatus = False
                                            updateMap['subOrders.$.imgUrl'] = imgUrl
                                            updateMap['subOrders.$.lastTracked'] = timestamp
                                            updateMap['subOrders.$.detailedStatus'] = orderStatus
                                            cashbackStatus = subOrder.get("cashBackStatus")
                                            updateMap['subOrders.$.status'] = status 
                                        
                                            if status==MStore.ORDER_DELIVERED:                               
                                                updateMap['subOrders.$.deliveredOn'] = deliveredOn
                                                closedStatus = True
                                                updateMap['subOrders.$.closed'] = True
                                                if cashbackStatus == Store.CB_PENDING:
                                                    updateMap['subOrders.$.cashbackStatus'] = Store.CB_APPROVED
                                            if status==MStore.ORDER_CANCELLED:     
                                                closedStatus = True
                                                updateMap['subOrders.$.closed'] = True
                                                if cashbackStatus == Store.CB_PENDING:
                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_CANCELLED
                                            if status==MStore.ORDER_SHIPPED:   
                                                updateMap['subOrders.$.estimatedDeliveryDate'] = deliveryEstimate
                                                if trackingUrl is not None:
                                                    updateMap['subOrders.$.trackingUrl'] = trackingUrl
                                            if not closedStatus:
                                                closed = False
                                            bulk.find(findMap).update({'$set' : updateMap})
                                            break
                            bulk.find({'orderId': activeOrder['orderId']}).update({'$set':{'closed': closed}})            
                bulk.execute()
                return 'PARSED_SUCCESS'
            else:
                merchantOrderId = re.findall(r'https://www.amazon.in/gp/css/summary/edit.html\?orderID=(.*)?', url, re.IGNORECASE)[0] 
                merchantOrder = self.db.merchantOrder.find_one({"merchantOrderId":merchantOrderId})
                self.parseOrderRawHtml(merchantOrder['orderId'], merchantOrder['subTagId'], merchantOrder['userId'], rawHtml, url, False)['result']
                return 'PARSED_SUCCESS'
                pass
            return 'PARSED_SUCCESS_NO_ORDERS'
        except:
            traceback.print_exc()    
            return 'PARSED_FAILED'
            
    def _getStatusFromDetailedStatus(self, detailedStatus):
        if "ordered from" in detailedStatus.lower():
            return MStore.ORDER_PLACED 

        for key, value in self.orderStatusRegexMap.iteritems():
            if detailedStatus.lower() in value:
                return key
        
        print "Detailed Status need to be mapped"
        print "Found new order status", detailedStatus
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)


def main():
    store = getStore(1)
    #store.scrapeStoreOrders()    
    store.parseOrderRawHtml(1, 'saad', '21232211', readSSh('/home/amit/Downloads/710.html'), 'https://www.amazon.in/gp/css/summary/edit.html?orderID=12212')
    #store.trackOrdersForUser(8,'https://www.amazon.in/gp/css/order-history', readSSh('/tmp/User2/2015-03-03 00:29:40.165513'))
    #readSSh('snapdeal.csv')
def readSSh(fileName):
    try:
            str1 = open(fileName).read()
            return str1
    except:
        ssh_client = SSHClient()
        str1 = ""
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh_client.connect('104.200.25.40', 22, 'root', 'ecip$dtrMay2014')
        sftp_client = ssh_client.open_sftp()
        try:
            if not os.path.exists(os.path.dirname(fileName)):
                os.makedirs(os.path.dirname(fileName))
            sftp_client.get(fileName, fileName)
            try:
                str1 = open(fileName).read()
                return str1
            finally:
                pass
        except:
            "could not read"
        return str1


if __name__ == '__main__':
        main()