WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/scripts/SnapdealPriceScraping.py

 (options, args) = parser.parse_args()
 mc = MemCache(options.mongoHost)
+SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
 headers = {
            'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
             'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
             'Accept-Language' : 'en-US,en;q=0.8',
             'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
             data = toScrapMap.get(manualDeal['sku'])
             data['dealFlag'] = 1
             data['dealType'] = manualDeal['dealType']
             data['dealPoints'] = manualDeal['dealPoints']
             data['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
+    for val in toScrapMap.values():
+        updatePrices(val)
-    pool = ThreadPool(cpu_count() *2)
+#    pool = ThreadPool(cpu_count() *2)
-    offset = 0
+#    offset = 0
-    limit =100
+#    limit =100
-    while(offset<=len(toScrapMap.values())):
+#    while(offset<=len(toScrapMap.values())):
-        pool.map(updatePrices,toScrapMap.values()[offset:offset+limit])
+#        pool.map(updatePrices,toScrapMap.values()[offset:offset+limit])
-        offset = offset + limit
+#        offset = offset + limit
-    pool.close()
+#    pool.close()
-    pool.join()
+#    pool.join()
-    print "joining threads at %s"%(str(datetime.now()))
+#    print "joining threads at %s"%(str(datetime.now()))
 def updatePrices(data):
     if data['source_id']!=3:
         return
     except:
         pass
     url="http://www.snapdeal.com/acors/json/v2/gvbps?supc=%s&catUrl=&bn=&catId=175&start=0&count=10000"%(data['identifier'].strip())
     print url
-    time.sleep(1)
     lowestOfferPrice = 0
     instock = 0
     buyBoxPrice = 0
     isBuyBox = 1
     stock = 0
+    try:
-    req = urllib2.Request(url,headers=headers)
+        req = urllib2.Request(url,headers=headers)
-    response = urllib2.urlopen(req)
+        response = urllib2.urlopen(req)
+    except:
+        print "Unable to scrape %d"%(data['_id'])
+        return
     try:
         vendorInfo = json.load(response)
     except:
         return
     finally:
     print instock
     print "Lowest Offer Price for id %d is %d , stock is %d and stock count is %d" %(data['_id'],lowestOfferPrice,instock,stock)
     print "*************"
     if instock  == 1:
         get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':instock,'buyBoxFlag':isBuyBox}}, multi=True)
-        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':instock}}, multi=True)
+        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)
     else:
         get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':instock,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':isBuyBox}}, multi=True)
-        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock}}, multi=True)
+        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock,'codAvailable':data['codAvailable']}})
     try:
         recomputeDeal(data)
     except:
         print "Unable to compute deal for ",data['skuBundleId']
     get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
 def recomputeDeal(item):
     """Lets recompute deal for this bundle"""
-    print "Recomputing for bundleId",item.get('skuBundleId')
+    print "Recomputing for bundleId %d" %(item.get('skuBundleId'))
     skuBundleId = item['skuBundleId']
     similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
     bestPrice = float("inf")
     bestOne = None
     bestSellerPoints = 0
     toUpdate = []
+    prepaidBestPrice = float("inf")
+    prepaidBestOne = None
+    prepaidBestSellerPoints = 0
     for similarItem in similarItems:
-        if mc.get("negative_deals") is None:
-            populateNegativeDeals()
-#        try:
-#            cashBack = getCashBack(similarItem['_id'], similarItem['source_id'], similarItem['category_id'], mc, options.mongoHost)
-#            if not cashBack or cashBack.get('cash_back_status')!=1:
-#                pass
-#            else:
-#                if cashBack['cash_back_type'] ==1:
-#                    similarItem['available_price'] = similarItem['available_price'] - similarItem['available_price'] * float(cashBack['cash_back'])/100
-#                elif cashBack['cash_back_type'] ==2:
-#                    similarItem['available_price'] = similarItem['available_price'] - float(cashBack['cash_back'])
-#                else:
-#                    pass
-#        except Exception as cashBackEx:
-#            print cashBackEx
-#            print "Error calculating cashback."
         if similarItem['_id'] == item['_id']:
             try:
                 recomputePoints(item, similarItem)
             except:
                 traceback.print_exc()
+        if similarItem['codAvailable'] ==1:
+            if mc.get("negative_deals") is None:
+                populateNegativeDeals()
-        if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
+            if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
+                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
+                continue
+            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
-            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
+                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
-            continue
+                continue
-        if similarItem['available_price'] < bestPrice:
+            if similarItem['available_price'] < bestPrice:
-            bestOne = similarItem
+                bestOne = similarItem
-            bestPrice = similarItem['available_price']
+                bestPrice = similarItem['available_price']
-            bestSellerPoints = similarItem['bestSellerPoints']
+                bestSellerPoints = similarItem['bestSellerPoints']
-        elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
+            elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
-            bestOne = similarItem
+                bestOne = similarItem
-            bestPrice = similarItem['available_price']
+                bestPrice = similarItem['available_price']
-            bestSellerPoints = similarItem['bestSellerPoints']
+                bestSellerPoints = similarItem['bestSellerPoints']
+            else:
+                pass
         else:
+            if mc.get("negative_deals") is None:
+                populateNegativeDeals()
+            if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
+                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
+                continue
+            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
+                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
+                continue
+            if similarItem['available_price'] < prepaidBestPrice:
+                prepaidBestOne = similarItem
+                prepaidBestPrice = similarItem['available_price']
+                prepaidBestSellerPoints = similarItem['bestSellerPoints']
+            elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
+                prepaidBestOne = similarItem
+                prepaidBestPrice = similarItem['available_price']
+                prepaidBestSellerPoints = similarItem['bestSellerPoints']
+            else:
-            pass
+                pass
-    if bestOne is not None:
+    if bestOne is not None and prepaidBestOne is not None:
         for similarItem in similarItems:
             toUpdate.append(similarItem['_id'])
         toUpdate.remove(bestOne['_id'])
+        toUpdate.remove(prepaidBestOne['_id'])
-        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
+        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
+        get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
     if len(toUpdate) > 0:
-        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
+        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
 def main():
     populate()
 if __name__=='__main__':
     main()

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/scripts/SnapdealPriceScraping.py – Rev 15827 → 16019