| Line 19... |
Line 19... |
| 19 |
|
19 |
|
| 20 |
(options, args) = parser.parse_args()
|
20 |
(options, args) = parser.parse_args()
|
| 21 |
|
21 |
|
| 22 |
mc = MemCache(options.mongoHost)
|
22 |
mc = MemCache(options.mongoHost)
|
| 23 |
|
23 |
|
| 24 |
ignoreItems = [24105, 23177]
|
- |
|
| 25 |
|
24 |
|
| 26 |
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
|
25 |
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
|
| 27 |
|
26 |
|
| 28 |
def get_mongo_connection(host=options.mongoHost, port=27017):
|
27 |
def get_mongo_connection(host=options.mongoHost, port=27017):
|
| 29 |
global con
|
28 |
global con
|
| Line 43... |
Line 42... |
| 43 |
amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
|
42 |
amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
|
| 44 |
for data in amazonBestSellers:
|
43 |
for data in amazonBestSellers:
|
| 45 |
if not toScrapMap.has_key(data['_id']):
|
44 |
if not toScrapMap.has_key(data['_id']):
|
| 46 |
data['dealFlag'] = 0
|
45 |
data['dealFlag'] = 0
|
| 47 |
data['dealType'] = 0
|
46 |
data['dealType'] = 0
|
| 48 |
data['dealPoints'] = 0
|
- |
|
| 49 |
data['manualDealThresholdPrice'] = None
|
- |
|
| 50 |
toScrapMap[data['_id']] = data
|
47 |
toScrapMap[data['_id']] = data
|
| 51 |
dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':-100}}))
|
48 |
dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':-100}}))
|
| 52 |
for deal in dealFlagged:
|
49 |
for deal in dealFlagged:
|
| 53 |
if not toScrapMap.has_key(deal['_id']):
|
50 |
if not toScrapMap.has_key(deal['_id']):
|
| 54 |
data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
|
51 |
data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
|
| 55 |
data[0]['dealFlag'] = 0
|
52 |
data[0]['dealFlag'] = 0
|
| 56 |
data[0]['dealType'] = 0
|
53 |
data[0]['dealType'] = 0
|
| 57 |
data[0]['dealPoints'] = 0
|
- |
|
| 58 |
data[0]['manualDealThresholdPrice'] = None
|
- |
|
| 59 |
toScrapMap[deal['_id']] = data[0]
|
54 |
toScrapMap[deal['_id']] = data[0]
|
| 60 |
manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':1}))
|
55 |
manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':1}))
|
| 61 |
for manualDeal in manualDeals:
|
56 |
for manualDeal in manualDeals:
|
| 62 |
if not toScrapMap.has_key(manualDeal['sku']):
|
57 |
if not toScrapMap.has_key(manualDeal['sku']):
|
| 63 |
data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
|
58 |
data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
|
| 64 |
if len(data) > 0:
|
59 |
if len(data) > 0:
|
| 65 |
data[0]['dealFlag'] = 1
|
60 |
data[0]['dealFlag'] = 1
|
| 66 |
data[0]['dealType'] = manualDeal['dealType']
|
61 |
data[0]['dealType'] = manualDeal['dealType']
|
| 67 |
data[0]['dealPoints'] = manualDeal['dealPoints']
|
- |
|
| 68 |
data[0]['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
|
- |
|
| 69 |
toScrapMap[manualDeal['sku']] = data[0]
|
62 |
toScrapMap[manualDeal['sku']] = data[0]
|
| 70 |
else:
|
63 |
else:
|
| 71 |
data = toScrapMap.get(manualDeal['sku'])
|
64 |
data = toScrapMap.get(manualDeal['sku'])
|
| 72 |
data['dealFlag'] = 1
|
65 |
data['dealFlag'] = 1
|
| 73 |
data['dealType'] = manualDeal['dealType']
|
66 |
data['dealType'] = manualDeal['dealType']
|
| 74 |
data['dealPoints'] = manualDeal['dealPoints']
|
- |
|
| 75 |
data['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
|
- |
|
| 76 |
pool = ThreadPool(cpu_count() * 2)
|
67 |
pool = ThreadPool(cpu_count() * 2)
|
| 77 |
pool.map(scrapeAmazon,toScrapMap.values())
|
68 |
pool.map(scrapeAmazon,toScrapMap.values())
|
| 78 |
pool.close()
|
69 |
pool.close()
|
| 79 |
pool.join()
|
70 |
pool.join()
|
| 80 |
print "joining threads at %s"%(str(datetime.now()))
|
71 |
print "joining threads at %s"%(str(datetime.now()))
|
| Line 85... |
Line 76... |
| 85 |
dealScraping = False
|
76 |
dealScraping = False
|
| 86 |
print str(data['identifier'])
|
77 |
print str(data['identifier'])
|
| 87 |
if data['identifier'] is None or len(data['identifier'].strip())==0:
|
78 |
if data['identifier'] is None or len(data['identifier'].strip())==0:
|
| 88 |
return
|
79 |
return
|
| 89 |
|
80 |
|
| 90 |
if data['_id'] in ignoreItems:
|
81 |
if data.get('ignorePricing') ==1:
|
| 91 |
print "Ignored items returning for %d"%(data['_id'])
|
82 |
print "Ignored items returning for %d"%(data['_id'])
|
| 92 |
return
|
83 |
return
|
| 93 |
|
84 |
|
| 94 |
try:
|
85 |
try:
|
| 95 |
if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
|
86 |
if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
|
| Line 141... |
Line 132... |
| 141 |
|
132 |
|
| 142 |
def populateNegativeDeals():
|
133 |
def populateNegativeDeals():
|
| 143 |
negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
|
134 |
negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
|
| 144 |
mc.set("negative_deals", negativeDeals, 600)
|
135 |
mc.set("negative_deals", negativeDeals, 600)
|
| 145 |
|
136 |
|
| 146 |
def recomputePoints(item, deal):
|
137 |
#def recomputePoints(item, deal):
|
| 147 |
try:
|
138 |
# try:
|
| 148 |
if item.get('available_price') == deal['available_price']:
|
139 |
# if item.get('available_price') == deal['available_price']:
|
| 149 |
print "No need to compute points for %d , as price is still same" %(item['_id'])
|
140 |
# print "No need to compute points for %d , as price is still same" %(item['_id'])
|
| 150 |
raise
|
141 |
# raise
|
| 151 |
nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
|
142 |
# nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
|
| 152 |
except:
|
143 |
# except:
|
| 153 |
traceback.print_exc()
|
144 |
# traceback.print_exc()
|
| 154 |
nlcPoints = deal['nlcPoints']
|
145 |
# nlcPoints = deal['nlcPoints']
|
| - |
|
146 |
#
|
| - |
|
147 |
# bundleDealPoints = list(get_mongo_connection().Catalog.DealPoints.find({'skuBundleId':item['skuBundleId'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))
|
| - |
|
148 |
# if len(bundleDealPoints) > 0:
|
| 155 |
if item['manualDealThresholdPrice'] >= deal['available_price']:
|
149 |
# item['manualDealThresholdPrice'] = bundleDealPoints[0]['dealThresholdPrice']
|
| 156 |
dealPoints = item['dealPoints']
|
150 |
# dealPoints = bundleDealPoints[0]['dealPoints']
|
| 157 |
else:
|
151 |
# else:
|
| 158 |
dealPoints = 0
|
152 |
# dealPoints = 0
|
| - |
|
153 |
# item['manualDealThresholdPrice'] = None
|
| - |
|
154 |
#
|
| 159 |
get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
|
155 |
# get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
|
| 160 |
|
156 |
|
| 161 |
|
157 |
|
| 162 |
|
158 |
|
| 163 |
def recomputeDeal(item):
|
159 |
def recomputeDeal(item):
|
| 164 |
"""Lets recompute deal for this bundle"""
|
160 |
"""Lets recompute deal for this bundle"""
|
| Line 172... |
Line 168... |
| 172 |
toUpdate = []
|
168 |
toUpdate = []
|
| 173 |
prepaidBestPrice = float("inf")
|
169 |
prepaidBestPrice = float("inf")
|
| 174 |
prepaidBestOne = None
|
170 |
prepaidBestOne = None
|
| 175 |
prepaidBestSellerPoints = 0
|
171 |
prepaidBestSellerPoints = 0
|
| 176 |
for similarItem in similarItems:
|
172 |
for similarItem in similarItems:
|
| 177 |
if similarItem['_id'] == item['_id']:
|
- |
|
| 178 |
try:
|
- |
|
| 179 |
recomputePoints(item, similarItem)
|
- |
|
| 180 |
except:
|
- |
|
| 181 |
traceback.print_exc()
|
- |
|
| 182 |
if similarItem['codAvailable'] ==1:
|
173 |
if similarItem['codAvailable'] ==1:
|
| 183 |
if mc.get("negative_deals") is None:
|
174 |
if mc.get("negative_deals") is None:
|
| 184 |
populateNegativeDeals()
|
175 |
populateNegativeDeals()
|
| 185 |
if similarItem['in_stock'] == 0 or similarItem['_id'] in mc.get("negative_deals"):
|
176 |
if similarItem['in_stock'] == 0 or similarItem['_id'] in mc.get("negative_deals"):
|
| 186 |
get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
|
177 |
get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
|