| 15509 |
manish.sha |
1 |
'''
|
|
|
2 |
Created on Jan 15, 2015
|
|
|
3 |
|
|
|
4 |
@author: Manish
|
|
|
5 |
'''
|
|
|
6 |
from bs4 import BeautifulSoup
|
|
|
7 |
from bson.binary import Binary
|
|
|
8 |
from datetime import datetime, date, timedelta
|
|
|
9 |
from dtr import main
|
|
|
10 |
from dtr.dao import AffiliateInfo, Order, SubOrder
|
|
|
11 |
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
|
|
|
12 |
Store as MStore, ungzipResponse, tprint
|
|
|
13 |
from dtr.storage import Mongo
|
|
|
14 |
from dtr.storage.Mongo import getImgSrc
|
|
|
15 |
from dtr.utils.utils import fetchResponseUsingProxy
|
|
|
16 |
from pprint import pprint
|
|
|
17 |
from pymongo import MongoClient
|
|
|
18 |
import json
|
|
|
19 |
import pymongo
|
|
|
20 |
import re
|
|
|
21 |
import time
|
|
|
22 |
import traceback
|
|
|
23 |
import urllib
|
|
|
24 |
from urlparse import urlparse, parse_qs
|
|
|
25 |
|
|
|
26 |
USERNAME='profittill2@gmail.com'
|
|
|
27 |
PASSWORD='spice@2020'
|
|
|
28 |
AFFILIATE_URL='http://affiliate.snapdeal.com'
|
|
|
29 |
POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
|
|
|
30 |
ORDER_TRACK_URL='http://www.shopclues.com/index.php?dispatch=order_lookup.details'
|
|
|
31 |
CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'
|
|
|
32 |
BASE_URL= 'http://www.shopclues.com'
|
|
|
33 |
BASE_MURL= 'http://m.shopclues.com'
|
|
|
34 |
|
|
|
35 |
class Store(MStore):
|
|
|
36 |
|
|
|
37 |
'''
|
|
|
38 |
This is to map order statuses of our system to order statuses of snapdeal.
|
|
|
39 |
And our statuses will change accordingly.
|
|
|
40 |
|
|
|
41 |
'''
|
|
|
42 |
OrderStatusMap = {
|
|
|
43 |
MStore.ORDER_PLACED : ['payment successful', 'new order - cod confirmation pending', 'processing', 'quality check','on schedule', 'processing - pickup-initiated', 'processing - ready to dispatch'],
|
|
|
44 |
MStore.ORDER_DELIVERED : ['delivered', 'complete'],
|
|
|
45 |
MStore.ORDER_SHIPPED : ['in transit', 'dispatched','shipped','order handed to courier'],
|
|
|
46 |
MStore.ORDER_CANCELLED : ['payment failed', 'canceled', 'payment declined', 'order on hold - cancellation requested by customer', 'courier returned']
|
|
|
47 |
}
|
|
|
48 |
OrderStatusConfirmationMap= {
|
|
|
49 |
"Payment Successful" : "P",
|
|
|
50 |
"Order Declined" : "D",
|
|
|
51 |
"New Order - COD confirmation Pending" : "O"
|
|
|
52 |
}
|
|
|
53 |
|
|
|
54 |
CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE
|
|
|
55 |
'''
|
|
|
56 |
def scrapeAffiliate(self, startDate=None, endDate=None):
|
|
|
57 |
br = getBrowserObject()
|
|
|
58 |
br.open(AFFILIATE_URL)
|
|
|
59 |
br.select_form(nr=0)
|
|
|
60 |
br.form['data[User][password]'] = PASSWORD
|
|
|
61 |
br.form['data[User][email]'] = USERNAME
|
|
|
62 |
br.submit()
|
|
|
63 |
response = br.open(CONFIG_URL)
|
|
|
64 |
|
|
|
65 |
token = re.findall('"session_token":"(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
|
|
|
66 |
print token
|
|
|
67 |
allOffers = self._getAllOffers(br, token)
|
|
|
68 |
self._saveToAffiliate(allOffers)
|
|
|
69 |
'''
|
|
|
70 |
|
|
|
71 |
def _setLastSaleDate(self, saleDate):
|
|
|
72 |
self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})
|
|
|
73 |
|
|
|
74 |
def getName(self):
|
|
|
75 |
return "shopclues"
|
|
|
76 |
|
|
|
77 |
def _getLastSaleDate(self,):
|
|
|
78 |
lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})
|
|
|
79 |
if lastDaySaleObj is None:
|
|
|
80 |
return datetime.min
|
|
|
81 |
|
|
|
82 |
def _getStatusFromDetailedStatus(self, detailedStatus):
|
|
|
83 |
for key, value in Store.OrderStatusMap.iteritems():
|
|
|
84 |
if detailedStatus.lower() in value:
|
|
|
85 |
return key
|
|
|
86 |
print "Detailed Status need to be mapped", detailedStatus, self.store_id
|
|
|
87 |
return None
|
|
|
88 |
|
|
|
89 |
def _getSingleSubOrderMap(self, orderId, orderStatus, statusTime, soup, subTagId):
|
|
|
90 |
productDetailsMap = {}
|
|
|
91 |
orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
|
|
|
92 |
orderTable.pop(0)
|
|
|
93 |
productDetailsSubMap = {}
|
|
|
94 |
for orderTr in orderTable:
|
|
|
95 |
cols = orderTr.find_all('td')
|
|
|
96 |
product_details = cols[0].find_all('a')
|
|
|
97 |
#print product_details
|
|
|
98 |
productUrl = product_details[0].get('href')
|
|
|
99 |
productName = product_details[0].contents[0].strip()
|
|
|
100 |
quantity = int(cols[1].text.strip())
|
|
|
101 |
sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
|
|
|
102 |
discount = long(cols[3].text.strip().replace("Rs.",""))
|
|
|
103 |
subtotal = long(cols[4].text.strip().replace("Rs.",""))
|
|
|
104 |
productDetailsSubMap['productUrl']=BASE_MURL+productUrl
|
|
|
105 |
productDetailsSubMap['productName']=productName
|
|
|
106 |
productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL+'&order_id=' +str(orderId)+'&email_id='+ subTagId.split('$')[1]
|
|
|
107 |
productDetailsSubMap['sellingPrice']=sellingPrice
|
|
|
108 |
productDetailsSubMap['quantity']=quantity
|
|
|
109 |
productDetailsSubMap['discount']=discount
|
|
|
110 |
productDetailsSubMap['subtotal']=subtotal
|
|
|
111 |
productDetailsSubMap['parentOrderId']=orderId
|
|
|
112 |
br = getBrowserObject()
|
|
|
113 |
productPage = br.open(BASE_MURL+productUrl)
|
|
|
114 |
productPageHeaders = str(productPage.info()).split('\n')
|
|
|
115 |
productPage = ungzipResponse(productPage)
|
|
|
116 |
jsonProductResponse = None
|
|
|
117 |
for header in productPageHeaders:
|
|
|
118 |
header = header.split(':')
|
|
|
119 |
if header[0] == 'Content-Type' and 'json' in header[1]:
|
|
|
120 |
jsonProductResponse = json.loads(productPage)
|
|
|
121 |
productPageSoup= None
|
|
|
122 |
if jsonProductResponse is not None:
|
|
|
123 |
productPageSoup = BeautifulSoup(jsonProductResponse['text'])
|
|
|
124 |
else:
|
|
|
125 |
productPageSoup = BeautifulSoup(productPage)
|
|
|
126 |
productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
|
|
|
127 |
lengthProductCodeArr= len(productCodeArray)
|
|
|
128 |
productCode = productCodeArray[lengthProductCodeArr-2]
|
|
|
129 |
allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
|
|
|
130 |
productImgUrl = ''
|
|
|
131 |
if allproductImageTags is not None and len(allproductImageTags)>0:
|
|
|
132 |
productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
|
|
|
133 |
productDetailsSubMap['productCode']=productCode
|
|
|
134 |
productDetailsSubMap['imgUrl']=productImgUrl
|
|
|
135 |
|
|
|
136 |
productDetailsSubMap['subOrderStatus']=orderStatus
|
|
|
137 |
productDetailsSubMap['subOrderStatusTime']=statusTime
|
|
|
138 |
productDetailsMap[orderId]=productDetailsSubMap
|
|
|
139 |
return productDetailsMap
|
|
|
140 |
|
|
|
141 |
def _getMultiSubOrdersMap(self, orderId, soup):
|
|
|
142 |
orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
|
|
|
143 |
orderTable.pop(0)
|
|
|
144 |
productDetailsMap = {}
|
|
|
145 |
for orderTr in orderTable:
|
|
|
146 |
productDetailsSubMap = {}
|
|
|
147 |
cols = orderTr.find_all('td')
|
|
|
148 |
product_details = cols[0].find_all('a')
|
|
|
149 |
#print product_details
|
|
|
150 |
subOrderId= product_details[1].contents[0].strip()
|
|
|
151 |
productUrl = product_details[0].get('href')
|
|
|
152 |
productName = product_details[0].contents[0].strip()
|
|
|
153 |
subOrderTrackingUrl = product_details[1].get('href')
|
|
|
154 |
quantity = int(cols[1].text.strip())
|
|
|
155 |
sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
|
|
|
156 |
discount = long(cols[3].text.strip().replace("Rs.",""))
|
|
|
157 |
subtotal = long(cols[4].text.strip().replace("Rs.",""))
|
|
|
158 |
productDetailsSubMap['productUrl']=BASE_MURL+productUrl
|
|
|
159 |
productDetailsSubMap['productName']=productName
|
|
|
160 |
productDetailsSubMap['subOrderTrackingUrl']=BASE_URL+subOrderTrackingUrl
|
|
|
161 |
productDetailsSubMap['sellingPrice']=sellingPrice
|
|
|
162 |
productDetailsSubMap['quantity']=quantity
|
|
|
163 |
productDetailsSubMap['discount']=discount
|
|
|
164 |
productDetailsSubMap['subtotal']=subtotal
|
|
|
165 |
br = getBrowserObject()
|
|
|
166 |
productPage = br.open(BASE_MURL+productUrl)
|
|
|
167 |
productPageHeaders = str(productPage.info()).split('\n')
|
|
|
168 |
productPage = ungzipResponse(productPage)
|
|
|
169 |
jsonProductResponse = None
|
|
|
170 |
for header in productPageHeaders:
|
|
|
171 |
header = header.split(':')
|
|
|
172 |
if header[0] == 'Content-Type' and 'json' in header[1]:
|
|
|
173 |
jsonProductResponse = json.loads(productPage)
|
|
|
174 |
productPageSoup= None
|
|
|
175 |
if jsonProductResponse is not None:
|
|
|
176 |
productPageSoup = BeautifulSoup(jsonProductResponse['text'])
|
|
|
177 |
else:
|
|
|
178 |
productPageSoup = BeautifulSoup(productPage)
|
|
|
179 |
productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
|
|
|
180 |
lengthProductCodeArr= len(productCodeArray)
|
|
|
181 |
productCode = productCodeArray[lengthProductCodeArr-2]
|
|
|
182 |
allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
|
|
|
183 |
productImgUrl = ''
|
|
|
184 |
if allproductImageTags is not None and len(allproductImageTags)>0:
|
|
|
185 |
productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
|
|
|
186 |
productDetailsSubMap['productCode']=productCode
|
|
|
187 |
productDetailsSubMap['imgUrl']=productImgUrl
|
|
|
188 |
br1 = getBrowserObject()
|
|
|
189 |
orderTrackingPage = br1.open(BASE_URL+subOrderTrackingUrl)
|
|
|
190 |
headers = str(orderTrackingPage.info()).split('\n')
|
|
|
191 |
orderTrackingPage= ungzipResponse(orderTrackingPage)
|
|
|
192 |
jsonResponse = None
|
|
|
193 |
for header in headers:
|
|
|
194 |
header = header.split(':')
|
|
|
195 |
if header[0] == 'Content-Type' and 'json' in header[1]:
|
|
|
196 |
jsonResponse = json.loads(orderTrackingPage)
|
|
|
197 |
|
|
|
198 |
orderTrackingPageSoup = None
|
|
|
199 |
if jsonResponse is not None:
|
|
|
200 |
orderTrackingPageSoup = BeautifulSoup(str(jsonResponse['text']))
|
|
|
201 |
else:
|
|
|
202 |
orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)
|
|
|
203 |
subOrderStatusList = orderTrackingPageSoup.findAll(attrs={'class' : 'price ord_status'})
|
|
|
204 |
subOrderStatus = subOrderStatusList[0].contents[0].strip()
|
|
|
205 |
subOrderStatusTime= orderTrackingPageSoup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
|
|
|
206 |
productDetailsSubMap['subOrderStatus'] = subOrderStatus
|
|
|
207 |
productDetailsSubMap['subOrderStatusTime'] = subOrderStatusTime
|
|
|
208 |
productDetailsSubMap['parentOrderId']=orderId
|
|
|
209 |
|
|
|
210 |
productDetailsMap[subOrderId]=productDetailsSubMap
|
|
|
211 |
|
|
|
212 |
return productDetailsMap
|
|
|
213 |
|
|
|
214 |
def updateCashbackInSubOrders(self, subOrders):
|
|
|
215 |
for subOrder in subOrders:
|
|
|
216 |
cashbackStatus = Store.CB_NA
|
|
|
217 |
cashbackAmount = 0
|
|
|
218 |
percentage = 0
|
|
|
219 |
amount = subOrder.amountPaid
|
|
|
220 |
if amount > 0:
|
|
|
221 |
(cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)
|
|
|
222 |
if cashbackAmount > 0:
|
|
|
223 |
cashbackStatus = Store.CB_PENDING
|
|
|
224 |
subOrder.cashBackStatus = cashbackStatus
|
|
|
225 |
subOrder.cashBackAmount = cashbackAmount
|
|
|
226 |
subOrder.cashBackPercentage = percentage
|
|
|
227 |
return subOrders
|
|
|
228 |
|
|
|
229 |
def _parseOrders(self, orderId, subTagId, userId, page, orderSuccessUrl):
|
|
|
230 |
soup = BeautifulSoup(page)
|
|
|
231 |
productDetailsMap = {}
|
|
|
232 |
orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})
|
|
|
233 |
paymentFields = soup.findAll(attrs={'class' : 'box_paymentcalculations_row'})
|
|
|
234 |
orderDateList = soup.findAll(attrs={'class':'price ord_date'})
|
|
|
235 |
placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace(" ","")
|
|
|
236 |
ordersubtotal=0
|
|
|
237 |
ordercluebucks=0
|
|
|
238 |
ordershippingcost=0
|
|
|
239 |
ordertotal=0
|
|
|
240 |
|
|
|
241 |
for val in paymentFields:
|
|
|
242 |
for value in val.contents:
|
|
|
243 |
if value is not None:
|
|
|
244 |
if 'div' in str(value).strip():
|
|
|
245 |
if 'Subtotal' in value.text.strip():
|
|
|
246 |
ordersubtotal = long(val.contents[3].text.strip().replace("Rs.",""))
|
|
|
247 |
print ordersubtotal
|
|
|
248 |
if 'Shipping Cost' in value.text.strip():
|
|
|
249 |
ordershippingcost = long(val.contents[3].text.strip().replace("Rs.",""))
|
|
|
250 |
print ordershippingcost
|
|
|
251 |
if 'Clue' in value.text.strip():
|
|
|
252 |
ordercluebucks = long(val.contents[3].text.strip().replace("Rs.",""))
|
|
|
253 |
print ordercluebucks
|
|
|
254 |
if 'Total' in value.text.strip():
|
|
|
255 |
ordertotal = long(val.contents[3].text.strip().replace("Rs.",""))
|
|
|
256 |
print ordertotal
|
|
|
257 |
|
|
|
258 |
if orderStatusList is not None and len(orderStatusList)>0:
|
|
|
259 |
orderStatus = orderStatusList[0].contents[0].strip()
|
|
|
260 |
statusTime= soup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
|
|
|
261 |
productDetailsMap = self._getSingleSubOrderMap(orderId, orderStatus, statusTime, soup, subTagId)
|
|
|
262 |
else:
|
|
|
263 |
productDetailsMap = self._getMultiSubOrdersMap(orderId, soup)
|
|
|
264 |
|
|
|
265 |
merchantOrder = Order(orderId, userId, subTagId.split('$')[0], self.store_id, orderSuccessUrl)
|
|
|
266 |
merchantOrder.placedOn = placedOn
|
|
|
267 |
merchantOrder.merchantOrderId = orderId
|
|
|
268 |
merchantOrder.paidAmount = ordertotal
|
|
|
269 |
totalOrdersAmount = 0
|
|
|
270 |
totalDiscount = 0
|
|
|
271 |
subOrders= []
|
|
|
272 |
for key in productDetailsMap:
|
|
|
273 |
subOrderDetail = productDetailsMap.get(key)
|
|
|
274 |
totalOrdersAmount = totalOrdersAmount + (subOrderDetail['sellingPrice'] * subOrderDetail['quantity'])
|
|
|
275 |
totalDiscount = totalDiscount + (subOrderDetail['discount'] * subOrderDetail['quantity'])
|
|
|
276 |
subOrder = SubOrder(subOrderDetail['productName'], subOrderDetail['productUrl'], placedOn, subOrderDetail['subtotal'])
|
|
|
277 |
subOrder.merchantSubOrderId = key
|
|
|
278 |
subOrder.detailedStatus = subOrderDetail['subOrderStatus']
|
|
|
279 |
subOrder.imgUrl = subOrderDetail['imgUrl']
|
|
|
280 |
subOrder.offerDiscount = subOrderDetail['discount'] * subOrderDetail['quantity']
|
|
|
281 |
subOrder.unitPrice = subOrderDetail['sellingPrice']
|
|
|
282 |
subOrder.productCode = subOrderDetail['productCode']
|
|
|
283 |
subOrder.amountPaid = subOrderDetail['subtotal']
|
|
|
284 |
subOrder.quantity = subOrderDetail['quantity']
|
|
|
285 |
subOrders.append(subOrder)
|
|
|
286 |
print totalOrdersAmount, totalDiscount
|
|
|
287 |
merchantOrder.totalAmount = totalOrdersAmount
|
|
|
288 |
merchantOrder.discountApplied = totalDiscount
|
|
|
289 |
merchantOrder.deliveryCharges = ordershippingcost
|
|
|
290 |
merchantOrder.subOrders = self.updateCashbackInSubOrders(subOrders)
|
|
|
291 |
|
|
|
292 |
return merchantOrder
|
|
|
293 |
|
|
|
294 |
def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
|
|
|
295 |
#print merchantOrder
|
|
|
296 |
resp = {}
|
|
|
297 |
try:
|
|
|
298 |
br = getBrowserObject()
|
|
|
299 |
url = ORDER_TRACK_URL +'&order_id=' +str(orderId)+'&email_id='+ subTagId.split('$')[1]
|
|
|
300 |
page = br.open(url)
|
|
|
301 |
headers = str(page.info()).split('\n')
|
|
|
302 |
page = ungzipResponse(page)
|
|
|
303 |
jsonResponse = None
|
|
|
304 |
for header in headers:
|
|
|
305 |
header = header.split(':')
|
|
|
306 |
if header[0] == 'Content-Type' and 'json' in header[1]:
|
|
|
307 |
jsonResponse = json.loads(page)
|
|
|
308 |
if jsonResponse is not None:
|
|
|
309 |
page = jsonResponse['text']
|
|
|
310 |
|
|
|
311 |
merchantOrder = self._parseOrders(orderId, subTagId, userId, page, orderSuccessUrl)
|
|
|
312 |
|
|
|
313 |
merchantOrder.orderTrackingUrl = url
|
|
|
314 |
|
|
|
315 |
if self._saveToOrder(todict(merchantOrder)):
|
|
|
316 |
resp['result'] = 'ORDER_CREATED'
|
|
|
317 |
else:
|
|
|
318 |
resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
|
|
|
319 |
|
|
|
320 |
return resp
|
|
|
321 |
except:
|
|
|
322 |
print "Error occurred"
|
|
|
323 |
traceback.print_exc()
|
|
|
324 |
resp['result'] = 'ORDER_NOT_CREATED'
|
|
|
325 |
|
|
|
326 |
def parseSingleSubOrder(self, soup, emailId, subOrderId, subOrderStatus):
|
|
|
327 |
orderDateList = soup.findAll(attrs={'class':'price ord_date'})
|
|
|
328 |
placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace(" ","")
|
|
|
329 |
orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
|
|
|
330 |
orderTable.pop(0)
|
|
|
331 |
subOrders =[]
|
|
|
332 |
for orderTr in orderTable:
|
|
|
333 |
cols = orderTr.find_all('td')
|
|
|
334 |
product_details = cols[0].find_all('a')
|
|
|
335 |
#print product_details
|
|
|
336 |
productUrl = product_details[0].get('href')
|
|
|
337 |
productName = product_details[0].contents[0].strip()
|
|
|
338 |
quantity = int(cols[1].text.strip())
|
|
|
339 |
sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
|
|
|
340 |
discount = long(cols[3].text.strip().replace("Rs.",""))
|
|
|
341 |
subtotal = long(cols[4].text.strip().replace("Rs.",""))
|
|
|
342 |
br = getBrowserObject()
|
|
|
343 |
productPage = br.open(BASE_MURL+productUrl)
|
|
|
344 |
productPageHeaders = str(productPage.info()).split('\n')
|
|
|
345 |
productPage = ungzipResponse(productPage)
|
|
|
346 |
jsonProductResponse = None
|
|
|
347 |
for header in productPageHeaders:
|
|
|
348 |
header = header.split(':')
|
|
|
349 |
if header[0] == 'Content-Type' and 'json' in header[1]:
|
|
|
350 |
jsonProductResponse = json.loads(productPage)
|
|
|
351 |
productPageSoup= None
|
|
|
352 |
if jsonProductResponse is not None:
|
|
|
353 |
productPageSoup = BeautifulSoup(jsonProductResponse['text'])
|
|
|
354 |
else:
|
|
|
355 |
productPageSoup = BeautifulSoup(productPage)
|
|
|
356 |
productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
|
|
|
357 |
lengthProductCodeArr= len(productCodeArray)
|
|
|
358 |
productCode = productCodeArray[lengthProductCodeArr-2]
|
|
|
359 |
allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
|
|
|
360 |
productImgUrl = ''
|
|
|
361 |
if allproductImageTags is not None and len(allproductImageTags)>0:
|
|
|
362 |
productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
|
|
|
363 |
|
|
|
364 |
subOrder = SubOrder(productName, productUrl, placedOn, subtotal)
|
|
|
365 |
subOrder.merchantSubOrderId = subOrderId
|
|
|
366 |
subOrder.detailedStatus = subOrderStatus
|
|
|
367 |
subOrder.imgUrl = productImgUrl
|
|
|
368 |
subOrder.offerDiscount = discount*quantity
|
|
|
369 |
subOrder.unitPrice = sellingPrice
|
|
|
370 |
subOrder.productCode = productCode
|
|
|
371 |
subOrder.amountPaid = subtotal
|
|
|
372 |
subOrder.quantity = quantity
|
|
|
373 |
|
|
|
374 |
subOrders.append(subOrder)
|
|
|
375 |
|
|
|
376 |
subOrders = self.updateCashbackInSubOrders(subOrders)
|
|
|
377 |
return subOrders[0]
|
|
|
378 |
|
|
|
379 |
def parseMultiSubOrders(self, soup):
|
|
|
380 |
orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
|
|
|
381 |
orderTable.pop(0)
|
|
|
382 |
orderDateList = soup.findAll(attrs={'class':'price ord_date'})
|
|
|
383 |
placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace(" ","")
|
|
|
384 |
subOrders = []
|
|
|
385 |
for orderTr in orderTable:
|
|
|
386 |
productDetailsSubMap = {}
|
|
|
387 |
cols = orderTr.find_all('td')
|
|
|
388 |
product_details = cols[0].find_all('a')
|
|
|
389 |
#print product_details
|
|
|
390 |
subOrderId= product_details[1].contents[0].strip()
|
|
|
391 |
productUrl = product_details[0].get('href')
|
|
|
392 |
productName = product_details[0].contents[0].strip()
|
|
|
393 |
subOrderTrackingUrl = product_details[1].get('href')
|
|
|
394 |
quantity = int(cols[1].text.strip())
|
|
|
395 |
sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
|
|
|
396 |
discount = long(cols[3].text.strip().replace("Rs.",""))
|
|
|
397 |
subtotal = long(cols[4].text.strip().replace("Rs.",""))
|
|
|
398 |
br = getBrowserObject()
|
|
|
399 |
productPage = br.open(BASE_MURL+productUrl)
|
|
|
400 |
productPageHeaders = str(productPage.info()).split('\n')
|
|
|
401 |
productPage = ungzipResponse(productPage)
|
|
|
402 |
jsonProductResponse = None
|
|
|
403 |
for header in productPageHeaders:
|
|
|
404 |
header = header.split(':')
|
|
|
405 |
if header[0] == 'Content-Type' and 'json' in header[1]:
|
|
|
406 |
jsonProductResponse = json.loads(productPage)
|
|
|
407 |
productPageSoup= None
|
|
|
408 |
if jsonProductResponse is not None:
|
|
|
409 |
productPageSoup = BeautifulSoup(jsonProductResponse['text'])
|
|
|
410 |
else:
|
|
|
411 |
productPageSoup = BeautifulSoup(productPage)
|
|
|
412 |
productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
|
|
|
413 |
lengthProductCodeArr= len(productCodeArray)
|
|
|
414 |
productCode = productCodeArray[lengthProductCodeArr-2]
|
|
|
415 |
allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
|
|
|
416 |
productImgUrl = ''
|
|
|
417 |
if allproductImageTags is not None and len(allproductImageTags)>0:
|
|
|
418 |
productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
|
|
|
419 |
productDetailsSubMap['productCode']=productCode
|
|
|
420 |
productDetailsSubMap['imgUrl']=productImgUrl
|
|
|
421 |
br1 = getBrowserObject()
|
|
|
422 |
orderTrackingPage = br1.open(BASE_URL+subOrderTrackingUrl)
|
|
|
423 |
headers = str(orderTrackingPage.info()).split('\n')
|
|
|
424 |
orderTrackingPage= ungzipResponse(orderTrackingPage)
|
|
|
425 |
jsonResponse = None
|
|
|
426 |
for header in headers:
|
|
|
427 |
header = header.split(':')
|
|
|
428 |
if header[0] == 'Content-Type' and 'json' in header[1]:
|
|
|
429 |
jsonResponse = json.loads(orderTrackingPage)
|
|
|
430 |
|
|
|
431 |
orderTrackingPageSoup = None
|
|
|
432 |
if jsonResponse is not None:
|
|
|
433 |
orderTrackingPageSoup = BeautifulSoup(str(jsonResponse['text']))
|
|
|
434 |
else:
|
|
|
435 |
orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)
|
|
|
436 |
subOrderStatusList = orderTrackingPageSoup.findAll(attrs={'class' : 'price ord_status'})
|
|
|
437 |
subOrderStatus = subOrderStatusList[0].contents[0].strip()
|
|
|
438 |
subOrderStatusTime= orderTrackingPageSoup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
|
|
|
439 |
|
|
|
440 |
subOrder = SubOrder(productName, productUrl, placedOn, subtotal)
|
|
|
441 |
subOrder.merchantSubOrderId = subOrderId
|
|
|
442 |
subOrder.detailedStatus = subOrderStatus
|
|
|
443 |
subOrder.imgUrl = productImgUrl
|
|
|
444 |
subOrder.offerDiscount = discount*quantity
|
|
|
445 |
subOrder.unitPrice = sellingPrice
|
|
|
446 |
subOrder.productCode = productCode
|
|
|
447 |
subOrder.amountPaid = subtotal
|
|
|
448 |
subOrder.quantity = quantity
|
|
|
449 |
|
|
|
450 |
subOrders.append(subOrder)
|
|
|
451 |
|
|
|
452 |
return self.updateCashbackInSubOrders(subOrders)
|
|
|
453 |
|
|
|
454 |
def scrapeStoreOrders(self,):
|
|
|
455 |
#collectionMap = {'palcedOn':1}
|
|
|
456 |
searchMap = {}
|
|
|
457 |
collectionMap = {"orderTrackingUrl":1}
|
|
|
458 |
orders = self._getActiveOrders(searchMap,collectionMap)
|
|
|
459 |
for order in orders:
|
|
|
460 |
print "Order", self.store_name, order['orderId'], order['orderTrackingUrl']
|
|
|
461 |
url = order['orderTrackingUrl']
|
|
|
462 |
page = fetchResponseUsingProxy(url)
|
|
|
463 |
soup = BeautifulSoup(page)
|
|
|
464 |
bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
|
|
|
465 |
closed = True
|
|
|
466 |
orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})
|
|
|
467 |
if orderStatusList is not None and len(orderStatusList)>0:
|
|
|
468 |
subOrderId = soup.findAll(attrs={'class':'price ord_no'})[0].text.strip()
|
|
|
469 |
subOrder = self._isSubOrderActive(order, subOrderId)
|
|
|
470 |
orderStatus = orderStatusList[0].contents[0].strip()
|
|
|
471 |
if subOrder is None:
|
|
|
472 |
try:
|
|
|
473 |
subOrder = self.parseSingleSubOrder(soup, order['orderTrackingUrl'].split('email_id=')[1], subOrderId, orderStatus)
|
|
|
474 |
if subOrder is None:
|
|
|
475 |
continue
|
|
|
476 |
self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrder])}}})
|
|
|
477 |
print "Added new suborders to Order id - ", order['orderId']
|
|
|
478 |
closed = False
|
|
|
479 |
except:
|
|
|
480 |
pass
|
|
|
481 |
continue
|
|
|
482 |
elif subOrder['closed']:
|
|
|
483 |
continue
|
|
|
484 |
else:
|
|
|
485 |
findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}
|
|
|
486 |
updateMap = {}
|
|
|
487 |
updateMap["subOrders.$.detailedStatus"] = orderStatus
|
|
|
488 |
status = self._getStatusFromDetailedStatus(orderStatus)
|
|
|
489 |
closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
|
|
|
490 |
if status is not None:
|
|
|
491 |
updateMap["subOrders.$.status"] = status
|
|
|
492 |
if closedStatus:
|
|
|
493 |
#if status is closed then change the paybackStatus accordingly
|
|
|
494 |
updateMap["subOrders.$.closed"] = True
|
|
|
495 |
if status == Store.ORDER_DELIVERED:
|
|
|
496 |
if subOrder.get("cashBackStatus") == Store.CB_PENDING:
|
|
|
497 |
updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
|
|
|
498 |
elif status == Store.ORDER_CANCELLED:
|
|
|
499 |
if subOrder.get("cashBackStatus") == Store.CB_PENDING:
|
|
|
500 |
updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
|
|
|
501 |
else:
|
|
|
502 |
closed = False
|
|
|
503 |
bulk.find(findMap).update({'$set' : updateMap})
|
|
|
504 |
|
|
|
505 |
else:
|
|
|
506 |
subOrdersList = self.parseMultiSubOrders(soup)
|
|
|
507 |
for subOrderObj in subOrdersList:
|
|
|
508 |
subOrder = self._isSubOrderActive(order, subOrderObj.merchantSubOrderId)
|
|
|
509 |
if subOrder is None:
|
|
|
510 |
self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrderObj])}}})
|
|
|
511 |
print "Added new suborders to Order id - ", order['orderId']
|
|
|
512 |
closed = False
|
|
|
513 |
continue
|
|
|
514 |
elif subOrder['closed']:
|
|
|
515 |
continue
|
|
|
516 |
else:
|
|
|
517 |
findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}
|
|
|
518 |
updateMap = {}
|
|
|
519 |
updateMap["subOrders.$.detailedStatus"] = subOrderObj.detailedStatus
|
|
|
520 |
status = self._getStatusFromDetailedStatus(orderStatus)
|
|
|
521 |
closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
|
|
|
522 |
if status is not None:
|
|
|
523 |
updateMap["subOrders.$.status"] = status
|
|
|
524 |
if closedStatus:
|
|
|
525 |
#if status is closed then change the paybackStatus accordingly
|
|
|
526 |
updateMap["subOrders.$.closed"] = True
|
|
|
527 |
if status == Store.ORDER_DELIVERED:
|
|
|
528 |
if subOrder.get("cashBackStatus") == Store.CB_PENDING:
|
|
|
529 |
updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
|
|
|
530 |
elif status == Store.ORDER_CANCELLED:
|
|
|
531 |
if subOrder.get("cashBackStatus") == Store.CB_PENDING:
|
|
|
532 |
updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
|
|
|
533 |
else:
|
|
|
534 |
closed = False
|
|
|
535 |
|
|
|
536 |
bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
|
|
|
537 |
result = bulk.execute()
|
|
|
538 |
tprint(result)
|
|
|
539 |
|
|
|
540 |
def saveToAffiliate(self, offers):
|
|
|
541 |
raise NotImplementedError
|
|
|
542 |
|
|
|
543 |
def todict(obj, classkey=None):
|
|
|
544 |
if isinstance(obj, dict):
|
|
|
545 |
data = {}
|
|
|
546 |
for (k, v) in obj.items():
|
|
|
547 |
data[k] = todict(v, classkey)
|
|
|
548 |
return data
|
|
|
549 |
elif hasattr(obj, "_ast"):
|
|
|
550 |
return todict(obj._ast())
|
|
|
551 |
elif hasattr(obj, "__iter__"):
|
|
|
552 |
return [todict(v, classkey) for v in obj]
|
|
|
553 |
elif hasattr(obj, "__dict__"):
|
|
|
554 |
data = dict([(key, todict(value, classkey))
|
|
|
555 |
for key, value in obj.__dict__.iteritems()
|
|
|
556 |
if not callable(value) and not key.startswith('_')])
|
|
|
557 |
if classkey is not None and hasattr(obj, "__class__"):
|
|
|
558 |
data[classkey] = obj.__class__.__name__
|
|
|
559 |
return data
|
|
|
560 |
else:
|
|
|
561 |
return obj
|
|
|
562 |
|