| Line 6... |
Line 6... |
| 6 |
from BeautifulSoup import BeautifulSoup
|
6 |
from BeautifulSoup import BeautifulSoup
|
| 7 |
from bson.binary import Binary
|
7 |
from bson.binary import Binary
|
| 8 |
from datetime import datetime, date, timedelta
|
8 |
from datetime import datetime, date, timedelta
|
| 9 |
from dtr import main
|
9 |
from dtr import main
|
| 10 |
from dtr.dao import AffiliateInfo, Order, SubOrder
|
10 |
from dtr.dao import AffiliateInfo, Order, SubOrder
|
| 11 |
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, Store as MStore,\
|
11 |
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
|
| 12 |
ungzipResponse
|
12 |
Store as MStore, ungzipResponse, tprint
|
| 13 |
from pprint import pprint
|
13 |
from pprint import pprint
|
| 14 |
from pymongo import MongoClient
|
14 |
from pymongo import MongoClient
|
| 15 |
import json
|
15 |
import json
|
| 16 |
import pymongo
|
16 |
import pymongo
|
| 17 |
import re
|
17 |
import re
|
| Line 239... |
Line 239... |
| 239 |
|
239 |
|
| 240 |
|
240 |
|
| 241 |
def scrapeStoreOrders(self,):
|
241 |
def scrapeStoreOrders(self,):
|
| 242 |
#collectionMap = {'palcedOn':1}
|
242 |
#collectionMap = {'palcedOn':1}
|
| 243 |
orders = self._getActiveOrders()
|
243 |
orders = self._getActiveOrders()
|
| 244 |
print "Found orders", orders
|
- |
|
| 245 |
br = getBrowserObject()
|
244 |
br = getBrowserObject()
|
| 246 |
for order in orders:
|
245 |
for order in orders:
|
| - |
|
246 |
print "Found order ", orders
|
| - |
|
247 |
try:
|
| 247 |
url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
|
248 |
url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
|
| 248 |
response = br.open(url)
|
249 |
response = br.open(url)
|
| 249 |
page = ungzipResponse(response)
|
250 |
page = ungzipResponse(response)
|
| 250 |
#page=page.decode("utf-8")
|
251 |
#page=page.decode("utf-8")
|
| 251 |
soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
|
252 |
soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
|
| 252 |
sections = soup.findAll("section")
|
253 |
sections = soup.findAll("section")
|
| 253 |
orderEl = sections[1]
|
254 |
orderEl = sections[1]
|
| 254 |
orderTrs = orderEl.findAll("tr")
|
255 |
orderTrs = orderEl.findAll("tr")
|
| 255 |
|
256 |
|
| 256 |
placedOn = str(orderTrs[0].findAll("td")[1].text)
|
257 |
placedOn = str(orderTrs[0].findAll("td")[1].text)
|
| 257 |
sections.pop(0)
|
258 |
sections.pop(0)
|
| 258 |
sections.pop(0)
|
259 |
sections.pop(0)
|
| 259 |
|
260 |
|
| 260 |
subOrders = sections
|
261 |
subOrders = sections
|
| 261 |
bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
|
262 |
bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
|
| 262 |
for subOrderElement in subOrders:
|
263 |
for subOrderElement in subOrders:
|
| 263 |
closed = True
|
264 |
closed = True
|
| 264 |
divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
|
265 |
divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
|
| 265 |
if len(divs)<=0:
|
266 |
if len(divs)<=0:
|
| 266 |
raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
|
267 |
raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
|
| 267 |
subOrder = None
|
268 |
subOrder = None
|
| 268 |
breakFlag = False
|
269 |
breakFlag = False
|
| 269 |
for div in divs:
|
270 |
for div in divs:
|
| 270 |
divStr = str(div)
|
271 |
divStr = str(div)
|
| 271 |
divStr = divStr.replace("\n","").replace("\t", "")
|
272 |
divStr = divStr.replace("\n","").replace("\t", "")
|
| 272 |
updateMap = {}
|
273 |
updateMap = {}
|
| 273 |
for line in divStr.split("<br />"):
|
274 |
for line in divStr.split("<br />"):
|
| 274 |
if "Suborder ID" in line:
|
275 |
if "Suborder ID" in line:
|
| 275 |
merchantSubOrderId = re.findall(r'\d+', line)[0]
|
276 |
merchantSubOrderId = re.findall(r'\d+', line)[0]
|
| 276 |
#break if suborder is inactive
|
277 |
#break if suborder is inactive
|
| 277 |
subOrder = self._isSubOrderActive(order, merchantSubOrderId)
|
278 |
subOrder = self._isSubOrderActive(order, merchantSubOrderId)
|
| 278 |
if subOrder is None:
|
279 |
if subOrder is None:
|
| 279 |
subOrders = self.parseSubOrder(subOrderElement, placedOn)
|
280 |
subOrders = self.parseSubOrder(subOrderElement, placedOn)
|
| 280 |
self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict(subOrders)}}})
|
281 |
self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict(subOrders)}}})
|
| 281 |
print "Added new suborders to Order id - " + order['orderId']
|
282 |
print "Added new suborders to Order id - " + order['orderId']
|
| 282 |
closed = False
|
283 |
closed = False
|
| 283 |
breakFlag = True
|
284 |
breakFlag = True
|
| 284 |
break
|
285 |
break
|
| 285 |
elif subOrder['closed']:
|
286 |
elif subOrder['closed']:
|
| 286 |
breakFlag = True
|
287 |
breakFlag = True
|
| 287 |
break
|
288 |
break
|
| 288 |
else:
|
289 |
else:
|
| 289 |
findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
|
290 |
findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
|
| 290 |
elif "Status" in line:
|
291 |
elif "Status" in line:
|
| 291 |
detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
|
292 |
detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
|
| 292 |
updateMap["subOrders.$.detailedStatus"] = detailedStatus
|
293 |
updateMap["subOrders.$.detailedStatus"] = detailedStatus
|
| 293 |
status = self._getStatusFromDetailedStatus(detailedStatus)
|
294 |
status = self._getStatusFromDetailedStatus(detailedStatus)
|
| 294 |
closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
|
295 |
closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
|
| 295 |
updateMap["subOrders.$.status"] = status
|
296 |
updateMap["subOrders.$.status"] = status
|
| 296 |
if detailedStatus == 'Closed For Vendor Reallocation':
|
297 |
if detailedStatus == 'Closed For Vendor Reallocation':
|
| 297 |
#if it is more than 6hours mark closed.
|
298 |
#if it is more than 6hours mark closed.
|
| 298 |
closeAt = subOrder.get("closeAt")
|
299 |
closeAt = subOrder.get("closeAt")
|
| 299 |
if closeAt is None:
|
300 |
if closeAt is None:
|
| 300 |
closeAt = datetime.now() + timedelta(hours=6)
|
301 |
closeAt = datetime.now() + timedelta(hours=6)
|
| 301 |
updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")
|
302 |
updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")
|
| - |
|
303 |
else:
|
| - |
|
304 |
closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")
|
| - |
|
305 |
if datetime.now() > closeAt:
|
| - |
|
306 |
closedStatus = True
|
| - |
|
307 |
|
| - |
|
308 |
|
| - |
|
309 |
if closedStatus:
|
| - |
|
310 |
#if status is closed then change the paybackStatus accordingly
|
| - |
|
311 |
updateMap["subOrders.$.closed"] = True
|
| - |
|
312 |
if status == Store.ORDER_DELIVERED:
|
| - |
|
313 |
if subOrder.get("cashBackStatus") == Store.CB_PENDING:
|
| - |
|
314 |
updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
|
| - |
|
315 |
elif status == Store.ORDER_CANCELLED:
|
| - |
|
316 |
if subOrder.get("cashBackStatus") == Store.CB_PENDING:
|
| - |
|
317 |
updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
|
| - |
|
318 |
|
| 302 |
else:
|
319 |
else:
|
| 303 |
closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")
|
- |
|
| 304 |
if datetime.now() > closeAt:
|
- |
|
| 305 |
closedStatus = True
|
- |
|
| 306 |
|
- |
|
| 307 |
|
- |
|
| 308 |
if closedStatus:
|
- |
|
| 309 |
#if status is closed then change the paybackStatus accordingly
|
- |
|
| 310 |
updateMap["subOrders.$.closed"] = True
|
- |
|
| 311 |
if status == Store.ORDER_DELIVERED:
|
- |
|
| 312 |
if subOrder.get("cashBackStatus") == Store.CB_PENDING:
|
- |
|
| 313 |
updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
|
- |
|
| 314 |
elif status == Store.ORDER_CANCELLED:
|
- |
|
| 315 |
if subOrder.get("cashBackStatus") == Store.CB_PENDING:
|
- |
|
| 316 |
updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
|
- |
|
| 317 |
|
- |
|
| 318 |
else:
|
- |
|
| 319 |
closed = False
|
320 |
closed = False
|
| 320 |
elif "Est. Shipping Date" in line:
|
321 |
elif "Est. Shipping Date" in line:
|
| 321 |
estimatedShippingDate = line.split(":")[1].strip()
|
322 |
estimatedShippingDate = line.split(":")[1].strip()
|
| 322 |
updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
|
323 |
updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
|
| 323 |
elif "Est. Delivery Date" in line:
|
324 |
elif "Est. Delivery Date" in line:
|
| 324 |
estimatedDeliveryDate = line.split(":")[1].strip()
|
325 |
estimatedDeliveryDate = line.split(":")[1].strip()
|
| 325 |
updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
|
326 |
updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
|
| 326 |
elif "Courier Name" in line:
|
327 |
elif "Courier Name" in line:
|
| 327 |
courierName = line.split(":")[1].strip()
|
328 |
courierName = line.split(":")[1].strip()
|
| 328 |
updateMap["subOrders.$.courierName"] = courierName
|
329 |
updateMap["subOrders.$.courierName"] = courierName
|
| 329 |
elif "Tracking No" in line:
|
330 |
elif "Tracking No" in line:
|
| 330 |
trackingNumber = line.split(":")[1].strip()
|
331 |
trackingNumber = line.split(":")[1].strip()
|
| 331 |
updateMap["subOrders.$.trackingNumber"] = trackingNumber
|
332 |
updateMap["subOrders.$.trackingNumber"] = trackingNumber
|
| 332 |
|
333 |
|
| 333 |
if breakFlag:
|
334 |
if breakFlag:
|
| 334 |
break
|
335 |
break
|
| 335 |
|
336 |
|
| 336 |
bulk.find(findMap).update({'$set' : updateMap})
|
337 |
bulk.find(findMap).update({'$set' : updateMap})
|
| 337 |
bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed}})
|
338 |
bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed}})
|
| 338 |
result = bulk.execute()
|
339 |
result = bulk.execute()
|
| 339 |
pprint(result)
|
340 |
pprint(result)
|
| - |
|
341 |
except:
|
| - |
|
342 |
tprint("Could not update " + str(order['orderId']))
|
| - |
|
343 |
traceback.print_exc()
|
| 340 |
|
344 |
|
| 341 |
|
345 |
|
| 342 |
def _saveToAffiliate(self, offers):
|
346 |
def _saveToAffiliate(self, offers):
|
| 343 |
if offers is None or len(offers)==0:
|
347 |
if offers is None or len(offers)==0:
|
| 344 |
print "no affiliate have been pushed"
|
348 |
print "no affiliate have been pushed"
|