| Line 20... |
Line 20... |
| 20 |
|
20 |
|
| 21 |
(options, args) = parser.parse_args()
|
21 |
(options, args) = parser.parse_args()
|
| 22 |
|
22 |
|
| 23 |
mc = MemCache(options.mongoHost)
|
23 |
mc = MemCache(options.mongoHost)
|
| 24 |
|
24 |
|
| 25 |
ignoreItems = [23364, 477, 476]
|
- |
|
| 26 |
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
|
25 |
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
|
| 27 |
|
26 |
|
| 28 |
def get_mongo_connection(host=options.mongoHost, port=27017):
|
27 |
def get_mongo_connection(host=options.mongoHost, port=27017):
|
| 29 |
global con
|
28 |
global con
|
| 30 |
if con is None:
|
29 |
if con is None:
|
| Line 43... |
Line 42... |
| 43 |
amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
|
42 |
amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
|
| 44 |
for data in amazonBestSellers:
|
43 |
for data in amazonBestSellers:
|
| 45 |
if not toScrapMap.has_key(data['_id']):
|
44 |
if not toScrapMap.has_key(data['_id']):
|
| 46 |
data['dealFlag'] = 0
|
45 |
data['dealFlag'] = 0
|
| 47 |
data['dealType'] = 0
|
46 |
data['dealType'] = 0
|
| 48 |
data['dealPoints'] = 0
|
- |
|
| 49 |
data['manualDealThresholdPrice'] = None
|
- |
|
| 50 |
toScrapMap[data['_id']] = data
|
47 |
toScrapMap[data['_id']] = data
|
| 51 |
dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':2,'showDeal':1,'totalPoints':{'$gt':-100}}))
|
48 |
dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':2,'showDeal':1,'totalPoints':{'$gt':-100}}))
|
| 52 |
for deal in dealFlagged:
|
49 |
for deal in dealFlagged:
|
| 53 |
if not toScrapMap.has_key(deal['_id']):
|
50 |
if not toScrapMap.has_key(deal['_id']):
|
| 54 |
data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
|
51 |
data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
|
| 55 |
data[0]['dealFlag'] = 0
|
52 |
data[0]['dealFlag'] = 0
|
| 56 |
data[0]['dealType'] = 0
|
53 |
data[0]['dealType'] = 0
|
| 57 |
data[0]['dealPoints'] = 0
|
- |
|
| 58 |
data[0]['manualDealThresholdPrice'] = None
|
- |
|
| 59 |
toScrapMap[deal['_id']] = data[0]
|
54 |
toScrapMap[deal['_id']] = data[0]
|
| 60 |
manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':2}))
|
55 |
manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':2}))
|
| 61 |
for manualDeal in manualDeals:
|
56 |
for manualDeal in manualDeals:
|
| 62 |
if not toScrapMap.has_key(manualDeal['sku']):
|
57 |
if not toScrapMap.has_key(manualDeal['sku']):
|
| 63 |
data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
|
58 |
data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
|
| 64 |
if len(data) > 0:
|
59 |
if len(data) > 0:
|
| 65 |
data[0]['dealFlag'] = 1
|
60 |
data[0]['dealFlag'] = 1
|
| 66 |
data[0]['dealType'] = manualDeal['dealType']
|
61 |
data[0]['dealType'] = manualDeal['dealType']
|
| 67 |
data[0]['dealPoints'] = manualDeal['dealPoints']
|
- |
|
| 68 |
data[0]['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
|
- |
|
| 69 |
toScrapMap[manualDeal['sku']] = data[0]
|
62 |
toScrapMap[manualDeal['sku']] = data[0]
|
| 70 |
else:
|
63 |
else:
|
| 71 |
data = toScrapMap.get(manualDeal['sku'])
|
64 |
data = toScrapMap.get(manualDeal['sku'])
|
| 72 |
data['dealFlag'] = 1
|
65 |
data['dealFlag'] = 1
|
| 73 |
data['dealType'] = manualDeal['dealType']
|
66 |
data['dealType'] = manualDeal['dealType']
|
| 74 |
data['dealPoints'] = manualDeal['dealPoints']
|
- |
|
| 75 |
data['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
|
- |
|
| 76 |
pool = ThreadPool(cpu_count() *2)
|
67 |
pool = ThreadPool(cpu_count() *2)
|
| 77 |
pool.map(scrapeFlipkart,toScrapMap.values())
|
68 |
pool.map(scrapeFlipkart,toScrapMap.values())
|
| 78 |
pool.close()
|
69 |
pool.close()
|
| 79 |
pool.join()
|
70 |
pool.join()
|
| 80 |
print "joining threads at %s"%(str(datetime.now()))
|
71 |
print "joining threads at %s"%(str(datetime.now()))
|
| Line 85... |
Line 76... |
| 85 |
retryCount = 0
|
76 |
retryCount = 0
|
| 86 |
if data['identifier'] is None or len(data['identifier'].strip())==0:
|
77 |
if data['identifier'] is None or len(data['identifier'].strip())==0:
|
| 87 |
print "returning in valid identifier"
|
78 |
print "returning in valid identifier"
|
| 88 |
return
|
79 |
return
|
| 89 |
|
80 |
|
| 90 |
if data['_id'] in ignoreItems:
|
81 |
if data.get('ignorePricing') ==1:
|
| 91 |
print "Ignored items returning for %d"%(data['_id'])
|
82 |
print "Ignored items returning for %d"%(data['_id'])
|
| 92 |
return
|
83 |
return
|
| 93 |
|
84 |
|
| 94 |
try:
|
85 |
try:
|
| 95 |
if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
|
86 |
if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
|
| Line 161... |
Line 152... |
| 161 |
try:
|
152 |
try:
|
| 162 |
recomputeDeal(data)
|
153 |
recomputeDeal(data)
|
| 163 |
except:
|
154 |
except:
|
| 164 |
print "Unable to compute deal for %s"%(data['skuBundleId'])
|
155 |
print "Unable to compute deal for %s"%(data['skuBundleId'])
|
| 165 |
|
156 |
|
| 166 |
def recomputePoints(item, deal):
|
157 |
#def recomputePoints(item, deal):
|
| 167 |
try:
|
158 |
# try:
|
| 168 |
if item.get('available_price') == deal['available_price']:
|
159 |
# if item.get('available_price') == deal['available_price']:
|
| 169 |
print "No need to compute points for %d , as price is still same" %(item['_id'])
|
160 |
# print "No need to compute points for %d , as price is still same" %(item['_id'])
|
| 170 |
raise
|
161 |
# raise
|
| 171 |
nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
|
162 |
# nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
|
| 172 |
except:
|
163 |
# except:
|
| 173 |
print traceback.print_exc()
|
164 |
# print traceback.print_exc()
|
| 174 |
nlcPoints = deal['nlcPoints']
|
165 |
# nlcPoints = deal['nlcPoints']
|
| - |
|
166 |
#
|
| - |
|
167 |
# bundleDealPoints = list(get_mongo_connection().Catalog.DealPoints.find({'skuBundleId':item['skuBundleId'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))
|
| - |
|
168 |
# if len(bundleDealPoints) > 0:
|
| 175 |
if item['manualDealThresholdPrice'] >= deal['available_price']:
|
169 |
# item['manualDealThresholdPrice'] = bundleDealPoints[0]['dealThresholdPrice']
|
| 176 |
dealPoints = item['dealPoints']
|
170 |
# dealPoints = bundleDealPoints[0]['dealPoints']
|
| 177 |
else:
|
171 |
# else:
|
| 178 |
dealPoints = 0
|
172 |
# dealPoints = 0
|
| - |
|
173 |
# item['manualDealThresholdPrice'] = None
|
| - |
|
174 |
#
|
| 179 |
get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
|
175 |
# get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
|
| 180 |
|
176 |
|
| 181 |
def populateNegativeDeals():
|
177 |
def populateNegativeDeals():
|
| 182 |
negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
|
178 |
negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
|
| 183 |
mc.set("negative_deals", negativeDeals, 600)
|
179 |
mc.set("negative_deals", negativeDeals, 600)
|
| 184 |
|
180 |
|
| Line 194... |
Line 190... |
| 194 |
toUpdate = []
|
190 |
toUpdate = []
|
| 195 |
prepaidBestPrice = float("inf")
|
191 |
prepaidBestPrice = float("inf")
|
| 196 |
prepaidBestOne = None
|
192 |
prepaidBestOne = None
|
| 197 |
prepaidBestSellerPoints = 0
|
193 |
prepaidBestSellerPoints = 0
|
| 198 |
for similarItem in similarItems:
|
194 |
for similarItem in similarItems:
|
| 199 |
if similarItem['_id'] == item['_id']:
|
- |
|
| 200 |
try:
|
- |
|
| 201 |
recomputePoints(item, similarItem)
|
- |
|
| 202 |
except:
|
- |
|
| 203 |
traceback.print_exc()
|
- |
|
| 204 |
if similarItem['codAvailable'] ==1:
|
195 |
if similarItem['codAvailable'] ==1:
|
| 205 |
if mc.get("negative_deals") is None:
|
196 |
if mc.get("negative_deals") is None:
|
| 206 |
populateNegativeDeals()
|
197 |
populateNegativeDeals()
|
| 207 |
if similarItem['in_stock'] == 0 or similarItem['_id'] in mc.get("negative_deals"):
|
198 |
if similarItem['in_stock'] == 0 or similarItem['_id'] in mc.get("negative_deals"):
|
| 208 |
get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
|
199 |
get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
|