| 13828 |
kshitij.so |
1 |
import pymongo
|
|
|
2 |
from dtr.utils.utils import to_java_date
|
| 13914 |
kshitij.so |
3 |
from datetime import datetime, timedelta
|
| 13832 |
kshitij.so |
4 |
from dtr.utils import AmazonPriceOnlyScraper
|
| 13828 |
kshitij.so |
5 |
|
|
|
6 |
con = None
|
|
|
7 |
scraperAmazon = AmazonPriceOnlyScraper.AmazonScraper()
|
|
|
8 |
|
|
|
9 |
|
|
|
10 |
def get_mongo_connection(host='localhost', port=27017):
|
|
|
11 |
global con
|
|
|
12 |
if con is None:
|
|
|
13 |
print "Establishing connection %s host and port %d" %(host,port)
|
|
|
14 |
try:
|
|
|
15 |
con = pymongo.MongoClient(host, port)
|
|
|
16 |
except Exception, e:
|
|
|
17 |
print e
|
|
|
18 |
return None
|
|
|
19 |
return con
|
|
|
20 |
|
|
|
21 |
def scrapeAmazon():
|
| 14133 |
kshitij.so |
22 |
bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
|
|
|
23 |
for bestSeller in bestSellers:
|
|
|
24 |
amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
|
|
|
25 |
for data in amazonBestSellers:
|
|
|
26 |
inStock = 0
|
|
|
27 |
print str(data['identifier'])
|
|
|
28 |
if data['identifier'] is None or len(data['identifier'].strip())==0:
|
|
|
29 |
print "continue"
|
| 13914 |
kshitij.so |
30 |
continue
|
|
|
31 |
|
| 14133 |
kshitij.so |
32 |
try:
|
|
|
33 |
if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
|
|
|
34 |
print "sku id is already updated",data['_id']
|
|
|
35 |
continue
|
|
|
36 |
except:
|
|
|
37 |
pass
|
|
|
38 |
|
|
|
39 |
url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data['identifier'].strip())
|
|
|
40 |
print url
|
|
|
41 |
lowestPrice = 0.0
|
|
|
42 |
lowestPrice = scraperAmazon.read(url)
|
|
|
43 |
print lowestPrice
|
|
|
44 |
if lowestPrice > 0:
|
|
|
45 |
inStock = 1
|
|
|
46 |
print lowestPrice
|
|
|
47 |
print inStock
|
|
|
48 |
if lowestPrice > 0:
|
|
|
49 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
|
|
|
50 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock}}, multi=True)
|
|
|
51 |
else:
|
|
|
52 |
get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
|
|
|
53 |
get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
|
|
|
54 |
|
|
|
55 |
try:
|
|
|
56 |
recomputeDeal(data['skuBundleId'])
|
|
|
57 |
except:
|
|
|
58 |
print "Unable to compute deal for ",data['skuBundleId']
|
| 13914 |
kshitij.so |
59 |
|
| 13828 |
kshitij.so |
60 |
|
| 13914 |
kshitij.so |
61 |
def recomputeDeal(skuBundleId):
|
|
|
62 |
"""Lets recompute deal for this bundle"""
|
|
|
63 |
print "Recomputing for bundleId",skuBundleId
|
|
|
64 |
|
|
|
65 |
similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
|
|
|
66 |
bestPrice = float("inf")
|
|
|
67 |
bestOne = None
|
|
|
68 |
bestSellerPoints = 0
|
|
|
69 |
toUpdate = []
|
|
|
70 |
for similarItem in similarItems:
|
| 13972 |
kshitij.so |
71 |
if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price']:
|
| 13914 |
kshitij.so |
72 |
get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
|
|
|
73 |
continue
|
|
|
74 |
if similarItem['available_price'] < bestPrice:
|
|
|
75 |
bestOne = similarItem
|
|
|
76 |
bestPrice = similarItem['available_price']
|
|
|
77 |
bestSellerPoints = similarItem['bestSellerPoints']
|
|
|
78 |
elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
|
|
|
79 |
bestOne = similarItem
|
|
|
80 |
bestPrice = similarItem['available_price']
|
|
|
81 |
bestSellerPoints = similarItem['bestSellerPoints']
|
|
|
82 |
else:
|
|
|
83 |
pass
|
|
|
84 |
if bestOne is not None:
|
|
|
85 |
for similarItem in similarItems:
|
|
|
86 |
toUpdate.append(similarItem['_id'])
|
|
|
87 |
toUpdate.remove(bestOne['_id'])
|
|
|
88 |
get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
|
|
|
89 |
if len(toUpdate) > 0:
|
|
|
90 |
get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
|
|
|
91 |
|
| 13828 |
kshitij.so |
92 |
def main():
|
|
|
93 |
scrapeAmazon()
|
|
|
94 |
|
|
|
95 |
if __name__=='__main__':
|
|
|
96 |
main()
|