WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/scripts/SnapdealPriceScraping.py

             'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
             'Accept-Language' : 'en-US,en;q=0.8',
             'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
+        }
-ignoreItems = [24291, 24102, 24101]
 def get_mongo_connection(host=options.mongoHost, port=27017):
     global con
     if con is None:
         print "Establishing connection %s host and port %d" %(host,port)
         snapdealBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':3}))
         for data in snapdealBestSellers:
             if not toScrapMap.has_key(data['_id']):
                 data['dealFlag'] = 0
                 data['dealType'] = 0
-                data['dealPoints'] = 0
-                data['manualDealThresholdPrice'] = None
                 toScrapMap[data['_id']] = data
     dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':3,'showDeal':1,'totalPoints':{'$gt':-100}}))
     for deal in dealFlagged:
         if not toScrapMap.has_key(deal['_id']):
             data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
             data[0]['dealFlag'] = 0
             data[0]['dealType'] = 0
-            data[0]['dealPoints'] = 0
-            data[0]['manualDealThresholdPrice'] = None
             toScrapMap[deal['_id']] = data[0]
     manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':3}))
     for manualDeal in manualDeals:
         if not toScrapMap.has_key(manualDeal['sku']):
             data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
             if len(data) > 0:
                 data[0]['dealFlag'] = 1
                 data[0]['dealType'] = manualDeal['dealType']
-                data[0]['dealPoints'] = manualDeal['dealPoints']
-                data[0]['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
                 toScrapMap[manualDeal['sku']] = data[0]
         else:
             data = toScrapMap.get(manualDeal['sku'])
             data['dealFlag'] = 1
             data['dealType'] = manualDeal['dealType']
-            data['dealPoints'] = manualDeal['dealPoints']
-            data['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
     for val in toScrapMap.values():
         updatePrices(val)
 #    pool = ThreadPool(cpu_count() *2)
 #    offset = 0
 #    limit =100
 #    pool.join()
 #    print "joining threads at %s"%(str(datetime.now()))
 def updatePrices(data):
-    if data['_id'] in ignoreItems:
+    if data.get('ignorePricing') ==1:
         print "Ignored items returning for %d"%(data['_id'])
         return
     if data['source_id']!=3:
         return
     print data['identifier']
 def populateNegativeDeals():
     negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
     mc.set("negative_deals", negativeDeals, 600)
-def recomputePoints(item, deal):
+#def recomputePoints(item, deal):
-    try:
+#    try:
+#        if item.get('available_price') == deal['available_price']:
+#            print "No need to compute points for %d , as price is still same" %(item['_id'])
+#            raise
-        nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
+#        nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
-    except:
+#    except:
-        traceback.print_exc()
+#        print traceback.print_exc()
-        nlcPoints = deal['nlcPoints']
+#        nlcPoints = deal['nlcPoints']
+#
+#    bundleDealPoints = list(get_mongo_connection().Catalog.DealPoints.find({'skuBundleId':item['skuBundleId'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))
+#    if len(bundleDealPoints) > 0:
-    if item['manualDealThresholdPrice'] >= deal['available_price']:
+#        item['manualDealThresholdPrice'] = bundleDealPoints[0]['dealThresholdPrice']
-        dealPoints = item['dealPoints']
+#        dealPoints = bundleDealPoints[0]['dealPoints']
-    else:
+#    else:
-        dealPoints = 0
+#        dealPoints = 0
+#        item['manualDealThresholdPrice'] = None
+#
-    get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
+#    get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
 def recomputeDeal(item):
     """Lets recompute deal for this bundle"""
     print "Recomputing for bundleId %d" %(item.get('skuBundleId'))
     toUpdate = []
     prepaidBestPrice = float("inf")
     prepaidBestOne = None
     prepaidBestSellerPoints = 0
     for similarItem in similarItems:
-        if similarItem['_id'] == item['_id']:
-            try:
-                recomputePoints(item, similarItem)
-            except:
-                traceback.print_exc()
         if similarItem['codAvailable'] ==1:
             if mc.get("negative_deals") is None:
                 populateNegativeDeals()
             if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                 get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/scripts/SnapdealPriceScraping.py – Rev 16429 → 16502