| Line 8... |
Line 8... |
| 8 |
from dtr.utils import AmazonDealScraper
|
8 |
from dtr.utils import AmazonDealScraper
|
| 9 |
from dtr.utils import FlipkartScraper,NewFlipkartScraper
|
9 |
from dtr.utils import FlipkartScraper,NewFlipkartScraper
|
| 10 |
from dtr.storage.MemCache import MemCache
|
10 |
from dtr.storage.MemCache import MemCache
|
| 11 |
from functools import partial
|
11 |
from functools import partial
|
| 12 |
import threading
|
12 |
import threading
|
| - |
|
13 |
from dtr.utils.utils import getCashBack
|
| 13 |
|
14 |
|
| 14 |
mc = MemCache("127.0.0.1")
|
15 |
mc = MemCache("127.0.0.1")
|
| 15 |
|
16 |
|
| 16 |
con = None
|
17 |
con = None
|
| 17 |
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4}
|
18 |
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4}
|
| Line 52... |
Line 53... |
| 52 |
pass
|
53 |
pass
|
| 53 |
|
54 |
|
| 54 |
|
55 |
|
| 55 |
url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data['identifier'])
|
56 |
url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data['identifier'])
|
| 56 |
lowestPrice = 0.0
|
57 |
lowestPrice = 0.0
|
| 57 |
print data['dealFlag']
|
- |
|
| 58 |
print data['dealType']
|
- |
|
| 59 |
try:
|
58 |
try:
|
| 60 |
if data['dealFlag'] ==1 and data['dealType'] ==1:
|
59 |
if data['dealFlag'] ==1 and data['dealType'] ==1:
|
| 61 |
print "Inside deal"
|
60 |
print "Inside deal"
|
| 62 |
deal_url = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())
|
61 |
deal_url = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())
|
| 63 |
print deal_url
|
62 |
print deal_url
|
| Line 83... |
Line 82... |
| 83 |
else:
|
82 |
else:
|
| 84 |
lowestPrice = data['available_price']
|
83 |
lowestPrice = data['available_price']
|
| 85 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)
|
84 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)
|
| 86 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType']}}, multi=True)
|
85 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType']}}, multi=True)
|
| 87 |
|
86 |
|
| 88 |
print "Returning info"
|
- |
|
| 89 |
return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
87 |
return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
| 90 |
except:
|
88 |
except:
|
| 91 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
89 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
| 92 |
|
90 |
|
| 93 |
elif source_id ==4:
|
91 |
elif source_id ==4:
|
| Line 120... |
Line 118... |
| 120 |
else:
|
118 |
else:
|
| 121 |
lowestPrice = data['available_price']
|
119 |
lowestPrice = data['available_price']
|
| 122 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
|
120 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
|
| 123 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock}}, multi=True)
|
121 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock}}, multi=True)
|
| 124 |
|
122 |
|
| 125 |
print "Returning info"
|
- |
|
| 126 |
return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
123 |
return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
| 127 |
|
124 |
|
| 128 |
except:
|
125 |
except:
|
| 129 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
126 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
| 130 |
|
127 |
|
| Line 169... |
Line 166... |
| 169 |
else:
|
166 |
else:
|
| 170 |
lowestOfferPrice = data['available_price']
|
167 |
lowestOfferPrice = data['available_price']
|
| 171 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
|
168 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
|
| 172 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
|
169 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
|
| 173 |
|
170 |
|
| 174 |
print "Returning info"
|
- |
|
| 175 |
return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
171 |
return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
| 176 |
except:
|
172 |
except:
|
| 177 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
173 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
| 178 |
|
174 |
|
| 179 |
elif source_id == 2:
|
175 |
elif source_id == 2:
|
| Line 220... |
Line 216... |
| 220 |
else:
|
216 |
else:
|
| 221 |
lowestSp = data['available_price']
|
217 |
lowestSp = data['available_price']
|
| 222 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
|
218 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
|
| 223 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
|
219 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
|
| 224 |
|
220 |
|
| 225 |
print "returning"
|
- |
|
| 226 |
return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
221 |
return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
| 227 |
|
222 |
|
| 228 |
except:
|
223 |
except:
|
| 229 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
224 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
|
| 230 |
else:
|
225 |
else:
|
| Line 277... |
Line 272... |
| 277 |
item['dealFlag'] = 0
|
272 |
item['dealFlag'] = 0
|
| 278 |
item['dealType'] = 0
|
273 |
item['dealType'] = 0
|
| 279 |
manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))
|
274 |
manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))
|
| 280 |
if len(manualDeals) > 0:
|
275 |
if len(manualDeals) > 0:
|
| 281 |
item['dealFlag'] = 1
|
276 |
item['dealFlag'] = 1
|
| 282 |
item['dealType'] =manualDeals[0]['dealType']
|
277 |
item['dealType'] =manualDeals[0]['dealType']
|
| 283 |
temp.append(returnLatestPrice(item, source_id))
|
278 |
info = returnLatestPrice(item, source_id)
|
| - |
|
279 |
try:
|
| - |
|
280 |
cashBack = getCashBack(item['_id'], item['source_id'], item['category_id'], mc, 'localhost')
|
| - |
|
281 |
if not cashBack or cashBack.get('cash_back_status')!=1:
|
| - |
|
282 |
pass
|
| - |
|
283 |
else:
|
| - |
|
284 |
if cashBack['cash_back_type'] in (1,2):
|
| - |
|
285 |
info['cash_back_type'] = cashBack['cash_back_type']
|
| - |
|
286 |
info['cash_back'] = float(cashBack['cash_back'])
|
| - |
|
287 |
else:
|
| - |
|
288 |
raise
|
| - |
|
289 |
except Exception as cashBackEx:
|
| - |
|
290 |
print cashBackEx
|
| - |
|
291 |
print "Error calculating cashback."
|
| - |
|
292 |
info['cash_back_type'] = 0
|
| - |
|
293 |
info['cash_back'] = 0
|
| - |
|
294 |
temp.append(info)
|
| 284 |
try:
|
295 |
try:
|
| 285 |
thread = threading.Thread(target=recomputeDeal, args = (skuBundleId,))
|
296 |
thread = threading.Thread(target=recomputeDeal, args = (skuBundleId,))
|
| 286 |
thread.daemon = True
|
297 |
thread.daemon = True
|
| 287 |
thread.start()
|
298 |
thread.start()
|
| 288 |
except:
|
299 |
except:
|
| Line 298... |
Line 309... |
| 298 |
item[0]['dealFlag'] = 1
|
309 |
item[0]['dealFlag'] = 1
|
| 299 |
item[0]['dealType'] =manualDeals[0]['dealType']
|
310 |
item[0]['dealType'] =manualDeals[0]['dealType']
|
| 300 |
|
311 |
|
| 301 |
info = returnLatestPrice(item[0], item[0]['source_id'])
|
312 |
info = returnLatestPrice(item[0], item[0]['source_id'])
|
| 302 |
try:
|
313 |
try:
|
| - |
|
314 |
cashBack = getCashBack(item['_id'], item['source_id'], item['category_id'], mc, 'localhost')
|
| - |
|
315 |
if not cashBack or cashBack.get('cash_back_status')!=1:
|
| - |
|
316 |
pass
|
| - |
|
317 |
else:
|
| - |
|
318 |
if cashBack['cash_back_type'] in (1,2):
|
| - |
|
319 |
info['cash_back_type'] = cashBack['cash_back_type']
|
| - |
|
320 |
info['cash_back'] = float(cashBack['cash_back'])
|
| - |
|
321 |
else:
|
| - |
|
322 |
raise
|
| - |
|
323 |
except Exception as cashBackEx:
|
| - |
|
324 |
print cashBackEx
|
| - |
|
325 |
print "Error calculating cashback."
|
| - |
|
326 |
info['cash_back_type'] = 0
|
| - |
|
327 |
info['cash_back'] = 0
|
| - |
|
328 |
try:
|
| 303 |
thread = threading.Thread(target=recomputeDeal, args = (item[0]['skuBundleId'],))
|
329 |
thread = threading.Thread(target=recomputeDeal, args = (item[0]['skuBundleId'],))
|
| 304 |
thread.daemon = True
|
330 |
thread.daemon = True
|
| 305 |
thread.start()
|
331 |
thread.start()
|
| 306 |
except:
|
332 |
except:
|
| 307 |
print "Unable to compute deal for ",item[0]['skuBundleId']
|
333 |
print "Unable to compute deal for ",item[0]['skuBundleId']
|
| 308 |
return info
|
334 |
return info
|
| 309 |
|
335 |
|
| 310 |
|
- |
|
| 311 |
def main():
|
336 |
def main():
|
| 312 |
print "retuned %s"%(str(getLatestPrice(17, 2)))
|
337 |
print "retuned %s"%(str(getLatestPrice(17, 2)))
|
| 313 |
|
338 |
|
| 314 |
if __name__=='__main__':
|
339 |
if __name__=='__main__':
|
| 315 |
main()
|
340 |
main()
|