Subversion Repositories SmartDukaan

Rev

Rev 14335 | Rev 14465 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14335 Rev 14464
Line 6... Line 6...
6 
'''
 6 
'''
7 
from base64 import encode
 7 
from base64 import encode
8 
from bs4 import BeautifulSoup
 8 
from bs4 import BeautifulSoup
9 
from datetime import datetime
 9 
from datetime import datetime
10 
from dtr.dao import Order, SubOrder
 10 
from dtr.dao import Order, SubOrder
11 
from dtr.main import getStore, Store as MStore, ParseException
 11 
from dtr.main import getStore, Store as MStore, ParseException, getBrowserObject, \
- 
 
 12 
    ungzipResponse
12 
from dtr.sources.flipkart import todict
 13 
from dtr.sources.flipkart import todict
13 
from paramiko import sftp
 14 
from paramiko import sftp
14 
from paramiko.client import SSHClient
 15 
from paramiko.client import SSHClient
15 
from paramiko.sftp_client import SFTPClient
 16 
from paramiko.sftp_client import SFTPClient
16 
import os.path
 17 
import os.path
Line 62... Line 63...
62 
                return resp
 63 
                return resp
63
64
64 
        else:
 65 
        else:
65 
            try:
 66 
            try:
66 
                merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
 67 
                merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
67 
                merchantOrder.orderTrackingUrl = orderSuccessUrl
 - 
 
68 
                soup = BeautifulSoup(rawHtml,from_encoding="utf-8")
 68 
                soup = BeautifulSoup(rawHtml)
69
- 
 
70 
                table = soup.body.findAll("table", recursive=False)[1]
 - 
 
71 
                #print table
 - 
 
72 
                tables = table.tr.td.findAll("table", recursive=False)
 - 
 
73 
                for tr in tables[2].findAll("tr"):
 - 
 
74 
                    boldElement = tr.td.b
 - 
 
75 
                    if "Order Placed" in str(boldElement):
 - 
 
76 
                        merchantOrder.placedOn = boldElement.next_sibling.strip()
 - 
 
77 
                    if "order number" in str(boldElement):
 - 
 
78 
                        merchantOrder.merchantOrderId = boldElement.next_sibling.strip()
 - 
 
79 
                    if "Order Total" in str(boldElement):
 - 
 
80 
                        merchantOrder.paidAmount = int(float(boldElement.find('span').contents[-1].replace(',','')))
 - 
 
81 
                anchors = table.tr.td.findAll("a", recursive=False)
 - 
 
82 
                paymentAnchor = anchors.pop(-1)
 - 
 
83
- 
 
84 
                count = 0
 - 
 
85 
                subOrders = []
 - 
 
86 
                merchantOrder.subOrders = subOrders
 - 
 
87 
                for anchor in anchors:
 - 
 
88 
                    count += 1
 - 
 
89 
                    tab = anchor.next_sibling
 - 
 
90 
                    status = MStore.ORDER_PLACED
 - 
 
91 
                    subStr = "Delivery #" + str(count) + ":"
 - 
 
92 
                    if subStr in  tab.find("b").text:
 - 
 
93 
                        detailedStatus = tab.find("b").text.replace(subStr, '').strip()
 - 
 
94
- 
 
95 
                    tab = tab.next_sibling.next_sibling
 - 
 
96 
                    trs = tab.find("table").find('tbody').findAll("tr", recursive = False)
 - 
 
97
- 
 
98 
                    estimatedDelivery = trs[0].td.find("b").next_sibling.strip()
 - 
 
99
69 
                try:
100 
                    orderItemTrs = trs[1].findAll("td", recursive=False)[1].table.tbody.findAll("tr", recursive = False)
 - 
 
101 
                    i = -1
 - 
 
102 
                    for orderItemTr in orderItemTrs:
 70 
                    self.parseOldStlye(merchantOrder, soup)
103 
                        i += 1
 - 
 
104 
                        if i%2 == 0:
 - 
 
105 
                            continue
 - 
 
106 
                        quantity =  int(re.findall(r'\d+', orderItemTr.td.contents[0])[0])
 - 
 
107
- 
 
108 
                        productUrl = orderItemTr.td.contents[1].a["href"]
 - 
 
109 
                        productTitle = orderItemTr.td.contents[1].a.text
 - 
 
110
- 
 
111 
                        unitPrice = int(float(orderItemTr.findAll('td')[1].span.text.replace('Rs. ','').replace(',','')))
 - 
 
112
- 
 
113
71 
                except:
114 
                        subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, unitPrice*quantity, status, quantity)
 - 
 
115 
                        subOrder.merchantSubOrderId = orderItemTr.find("input")["name"]
 - 
 
116 
                        subOrder.estimatedDeliveryDate = estimatedDelivery
 - 
 
117 
                        subOrder.productCode = productUrl.split('/')[5]
 - 
 
118 
                        subOrder.detailedStatus = detailedStatus
 - 
 
119 
                        (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, unitPrice*quantity)
 - 
 
120 
                        cashbackStatus = Store.CB_PENDING
 - 
 
121 
                        if cashbackAmount <= 0:
 - 
 
122 
                            cashbackStatus = Store.CB_NA
 - 
 
123 
                        subOrder.cashBackStatus = cashbackStatus
 - 
 
124 
                        subOrder.cashBackAmount = cashbackAmount
 - 
 
125 
                        if percentage > 0:
 - 
 
126 
                            subOrder.cashBackPercentage = percentage
 - 
 
127 
                        subOrders.append(subOrder)
 - 
 
128 
                priceList = paymentAnchor.next_sibling.next_sibling.next_sibling.table.table.tbody.tbody.tbody.findAll('tr', recursive=False)
 - 
 
129 
                totalAmount = 0
 - 
 
130 
                grandAmount = 0
 - 
 
131 
                for price in priceList:
 - 
 
132 
                    labelTd = price.td
 - 
 
133 
                    if 'Subtotal:' in labelTd.text:
 - 
 
134 
                        totalAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))
 - 
 
135 
                    elif 'Grand Total:' in labelTd.text:
 - 
 
136 
                        grandAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))
 - 
 
137 
                if grandAmount < totalAmount:
 - 
 
138 
                    diff = totalAmount - grandAmount
 - 
 
139 
                    for subOrder in merchantOrder.subOrders:
 - 
 
140 
                        subOrder.amountPaid -= int(diff*(1-subOrder.amountPaid/totalAmount))
 - 
 
141 
                self._updateToOrder(todict(merchantOrder))
 72 
                    self.parseNewStlye(merchantOrder, soup)
142 
                resp['result'] = 'ORDER_CREATED'
 73 
                resp['result'] = 'ORDER_CREATED'
143 
                return resp
74 
                return resp
144 
            except:
 75 
            except:
145 
                print "Error occurred"
 76 
                print "Error occurred"
146 
                traceback.print_exc()
 77 
                traceback.print_exc()
Line 149... Line 80...
149
80
150 
    #This should be exposed from api for specific sources
 81 
    #This should be exposed from api for specific sources
151 
    def scrapeStoreOrders(self):
 82 
    def scrapeStoreOrders(self):
152 
        pass
 83 
        pass
153
84
- 
 
 85 
    def parseOldStlye(self, merchantOrder, soup):
- 
 
 86 
        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl
- 
 
 87 
        table = soup.body.findAll("table", recursive=False)[1]
- 
 
 88 
        #print table
- 
 
 89 
        tables = table.tr.td.findAll("table", recursive=False)
- 
 
 90 
        for tr in tables[2].findAll("tr"):
- 
 
 91 
            boldElement = tr.td.b
- 
 
 92 
            if "Order Placed" in str(boldElement):
- 
 
 93 
                merchantOrder.placedOn = boldElement.next_sibling.strip()
- 
 
 94 
            if "order number" in str(boldElement):
- 
 
 95 
                merchantOrder.merchantOrderId = boldElement.next_sibling.strip()
- 
 
 96 
            if "Order Total" in str(boldElement):
- 
 
 97 
                merchantOrder.paidAmount = int(float(boldElement.find('span').contents[-1].replace(',','')))
- 
 
 98 
        anchors = table.tr.td.findAll("a", recursive=False)
- 
 
 99 
        paymentAnchor = anchors.pop(-1)
- 
 
 100
- 
 
 101 
        count = 0
- 
 
 102 
        subOrders = []
- 
 
 103 
        merchantOrder.subOrders = subOrders
- 
 
 104 
        counter = 0
- 
 
 105 
        for anchor in anchors:
- 
 
 106 
            count += 1
- 
 
 107 
            tab = anchor.next_sibling
- 
 
 108 
            status = MStore.ORDER_PLACED
- 
 
 109 
            subStr = "Delivery #" + str(count) + ":"
- 
 
 110 
            if subStr in  tab.find("b").text:
- 
 
 111 
                detailedStatus = tab.find("b").text.replace(subStr, '').strip()
- 
 
 112
- 
 
 113 
            tab = tab.next_sibling.next_sibling
- 
 
 114 
            trs = tab.find("table").find('tbody').findAll("tr", recursive = False)
- 
 
 115
- 
 
 116 
            estimatedDelivery = trs[0].td.find("b").next_sibling.strip()
- 
 
 117
- 
 
 118 
            orderItemTrs = trs[1].findAll("td", recursive=False)[1].table.tbody.findAll("tr", recursive = False)
- 
 
 119 
            i = -1
- 
 
 120 
            for orderItemTr in orderItemTrs:
- 
 
 121 
                i += 1
- 
 
 122 
                if i%2 == 0:
- 
 
 123 
                    continue
- 
 
 124 
                counter += 1
- 
 
 125 
                quantity =  int(re.findall(r'\d+', orderItemTr.td.contents[0])[0])
- 
 
 126
- 
 
 127 
                productUrl = orderItemTr.td.contents[1].a["href"]
- 
 
 128 
                productTitle = orderItemTr.td.contents[1].a.text
- 
 
 129
- 
 
 130 
                unitPrice = int(float(orderItemTr.findAll('td')[1].span.text.replace('Rs. ','').replace(',','')))
- 
 
 131
- 
 
 132
- 
 
 133 
                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, unitPrice*quantity, status, quantity)
- 
 
 134 
                subOrder.merchantSubOrderId = str(counter) + " of " + merchantOrder.merchantOrderId
- 
 
 135 
                subOrder.estimatedDeliveryDate = estimatedDelivery
- 
 
 136 
                subOrder.productCode = productUrl.split('/')[5]
- 
 
 137 
                subOrder.detailedStatus = detailedStatus
- 
 
 138 
                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, unitPrice*quantity)
- 
 
 139 
                cashbackStatus = Store.CB_PENDING
- 
 
 140 
                if cashbackAmount <= 0:
- 
 
 141 
                    cashbackStatus = Store.CB_NA
- 
 
 142 
                subOrder.cashBackStatus = cashbackStatus
- 
 
 143 
                subOrder.cashBackAmount = cashbackAmount
- 
 
 144 
                if percentage > 0:
- 
 
 145 
                    subOrder.cashBackPercentage = percentage
- 
 
 146 
                subOrders.append(subOrder)
- 
 
 147 
        priceList = paymentAnchor.next_sibling.next_sibling.next_sibling.table.table.tbody.tbody.tbody.findAll('tr', recursive=False)
- 
 
 148 
        totalAmount = 0
- 
 
 149 
        grandAmount = 0
- 
 
 150 
        for price in priceList:
- 
 
 151 
            labelTd = price.td
- 
 
 152 
            if 'Subtotal:' in labelTd.text:
- 
 
 153 
                totalAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))
- 
 
 154 
            elif 'Grand Total:' in labelTd.text:
- 
 
 155 
                grandAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))
- 
 
 156 
        if grandAmount < totalAmount:
- 
 
 157 
            diff = totalAmount - grandAmount
- 
 
 158 
            for subOrder in merchantOrder.subOrders:
- 
 
 159 
                subOrder.amountPaid -= int(diff*(1-subOrder.amountPaid/totalAmount))
- 
 
 160 
        self._updateToOrder(todict(merchantOrder))
- 
 
 161 
 
- 
 
 162 
    def parseNewStlye(self, merchantOrder, soup):
- 
 
 163 
        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl
- 
 
 164 
        orderDetailsContainer = soup.body.find(id="orderDetails")
- 
 
 165 
        orderLeftDiv = orderDetailsContainer.h1.next_sibling.next_sibling.div
- 
 
 166 
        placedOnSpan = orderLeftDiv.find("span", {'class':'order-date-invoice-item'})
- 
 
 167 
        merchantOrder.placedOn =placedOnSpan.text.split('Ordered on')[1].strip()
- 
 
 168 
        merchantOrder.merchantOrderId = placedOnSpan.next_sibling.next_sibling.text.split('Order#')[1].strip()
- 
 
 169 
        priceBox = orderDetailsContainer.find('div', {'class':re.compile(r'\ba-box-inner\b')}).div.div.findAll('div', recursive=False)[-1]
- 
 
 170 
        priceRows = priceBox.findAll('div', {'class':'a-row'})
- 
 
 171 
        subTotal = 0
- 
 
 172 
        shippingPrice = 0
- 
 
 173 
        promoApplied = 0
- 
 
 174 
        for priceRow in priceRows:
- 
 
 175 
            if "Item(s) Subtotal:" in str(priceRow):
- 
 
 176 
                subTotal = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
- 
 
 177 
            elif "Shipping:" in str(priceRow):
- 
 
 178 
                shippingPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
- 
 
 179 
            elif "Grand Total:" in str(priceRow):
- 
 
 180 
                grandPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
- 
 
 181 
                merchantOrder.paidAmount = grandPrice
- 
 
 182 
            elif "Total:" in str(priceRow):
- 
 
 183 
                totalPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
- 
 
 184 
            elif "Promotion Applied:" in str(priceRow):
- 
 
 185 
                promoApplied  += int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))
- 
 
 186 
        totalPaid = subTotal
- 
 
 187 
        if promoApplied > 0:
- 
 
 188 
            totalPaid -= promoApplied
- 
 
 189 
            if shippingPrice <= promoApplied:
- 
 
 190 
                totalPaid  += shippingPrice
- 
 
 191 
 
- 
 
 192 
        shipmentDivs = orderDetailsContainer.find('div', {'class':'a-box shipment'}).findAll('div', recursive = False)
- 
 
 193 
        subOrders = []
- 
 
 194 
        merchantOrder.subOrders = subOrders
- 
 
 195 
        i=1
- 
 
 196 
        for shipmentDiv in shipmentDivs:
- 
 
 197 
            innerBoxes = shipmentDiv.findAll('div', recursive = False)
- 
 
 198 
            statusDiv = innerBoxes[0]
- 
 
 199 
            subOrderStatus = statusDiv.div.span.text.strip()
- 
 
 200 
            estimatedDeliveryDate = statusDiv.div.div.find_next_sibling('div').span.span.text.strip()
- 
 
 201 
            productDivs = innerBoxes[-1].div.div.div.findAll('div', recursive=False)
- 
 
 202 
            subOrders = []
- 
 
 203 
            merchantOrder.subOrders = subOrders
- 
 
 204 
            for i, productDiv in enumerate(productDivs):
- 
 
 205 
                i +=1
- 
 
 206 
                imgDiv  = productDiv.div.div
- 
 
 207 
                detailDiv = imgDiv.find_next_sibling('div')
- 
 
 208 
                detailDivs = detailDiv.findAll('div', recursive=False)
- 
 
 209 
                arr = detailDivs[0].a.text.strip().split(" of ", 1)
- 
 
 210 
                (productTitle, quantity) = (arr[-1], (1 if type(arr[0])==str else int(arr[0])) )
- 
 
 211 
                unitPrice = int(float(detailDivs[2].span.text.replace('Rs. ','').replace(',','')))
- 
 
 212 
                amountPaid = int((unitPrice*quantity*totalPaid)/subTotal)
- 
 
 213 
                productUrl = "http://www.amazon.in" + detailDivs[0].a.get('href')
- 
 
 214 
                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid, MStore.ORDER_PLACED, quantity)
- 
 
 215 
                subOrder.productCode = productUrl.split('/')[5]
- 
 
 216 
                subOrder.unitPrice = unitPrice
- 
 
 217 
                subOrder.merchantSubOrderId = str(i) + " of " + merchantOrder.merchantOrderId
- 
 
 218 
                subOrder.estimatedDeliveryDate = estimatedDeliveryDate
- 
 
 219 
                subOrder.detailedStatus = subOrderStatus
- 
 
 220 
                subOrder.deliveryCharges = shippingPrice
- 
 
 221
- 
 
 222 
                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amountPaid)
- 
 
 223 
                cashbackStatus = Store.CB_PENDING
- 
 
 224 
                if cashbackAmount <= 0:
- 
 
 225 
                    cashbackStatus = Store.CB_NA
- 
 
 226 
                subOrder.cashBackStatus = cashbackStatus
- 
 
 227 
                subOrder.cashBackAmount = cashbackAmount
- 
 
 228 
                if percentage > 0:
- 
 
 229 
                    subOrder.cashBackPercentage = percentage
- 
 
 230 
                subOrders.append(subOrder)
- 
 
 231 
        self._updateToOrder(todict(merchantOrder))
- 
 
 232 
 
154 
    def getTrackingUrls(self, userId):
 233 
    def getTrackingUrls(self, userId):
155
234
156 
        missingOrderUrls = []
 235 
        missingOrderUrls = []
157 
        missingOrders = self._getMissingOrders({'userId':userId})
 236 
        missingOrders = self._getMissingOrders({'userId':userId})
158 
        for missingOrder in missingOrders:
 237 
        for missingOrder in missingOrders:
Line 276... Line 355...
276 
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
 355 
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
277 
 
 356 
 
278 
 
 357 
 
279 
def main():
 358 
def main():
280 
    store = getStore(1)
 359 
    store = getStore(1)
281
360 
    #store.scrapeStoreOrders()
282 
    #store.parseOrderRawHtml(1, 'saad', '2123221', readSSh('/tmp/User8/2015-03-02 12:51:59.545557'), 'https://www.amazon.in/gp/css/summary/edit.html?orderID=12212')
 361 
    store.parseOrderRawHtml(1, 'saad', '21232211', readSSh('/tmp/1.html'), 'https://www.amazon.in/gp/css/summary/edit.html?orderID=12212')
283 
    store.trackOrdersForUser(8,'https://www.amazon.in/gp/css/order-history', readSSh('/tmp/User2/2015-03-03 00:29:40.165513'))
 362 
    #store.trackOrdersForUser(8,'https://www.amazon.in/gp/css/order-history', readSSh('/tmp/User2/2015-03-03 00:29:40.165513'))
284 
 
 - 
 
- 
 
 363 
    #readSSh('snapdeal.csv')
285 
def readSSh(fileName):
 364 
def readSSh(fileName):
286 
    try:
 365 
    try:
287 
            str1 = open(fileName).read()
 366 
            str1 = open(fileName).read()
288 
            return str1
 367 
            return str1
289 
    except:
 368 
    except: