WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/scripts/FlipkartPriceScraping.py

 import pymongo
-from dtr.utils.utils import to_java_date
+from dtr.utils.utils import to_java_date, getNlcPoints
 from datetime import datetime, timedelta
 from operator import itemgetter
 from dtr.utils import FlipkartScraper,NewFlipkartScraper
 from multiprocessing import Pool as ThreadPool
 from multiprocessing import cpu_count
 import optparse
 from dtr.storage.MemCache import MemCache
 from dtr.utils.utils import getCashBack
+import traceback
 con = None
 parser = optparse.OptionParser()
 parser.add_option("-m", "--m", dest="mongoHost",
     bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
     for bestSeller in bestSellers:
         amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
         for data in amazonBestSellers:
             if not toScrapMap.has_key(data['_id']):
+                data['dealFlag'] = 0
+                data['dealType'] = 0
+                data['dealPoints'] = 0
+                data['manualDealThresholdPrice'] = None
                 toScrapMap[data['_id']] = data
     dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':2,'showDeal':1,'totalPoints':{'$gt':0}}))
     for deal in dealFlagged:
         if not toScrapMap.has_key(deal['_id']):
             data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
+            data[0]['dealFlag'] = 0
+            data[0]['dealType'] = 0
+            data[0]['dealPoints'] = 0
+            data[0]['manualDealThresholdPrice'] = None
             toScrapMap[deal['_id']] = data[0]
+    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':2}))
+    for manualDeal in manualDeals:
+        if not toScrapMap.has_key(manualDeal['sku']):
+            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
+            if len(data) > 0:
+                data[0]['dealFlag'] = 1
+                data[0]['dealType'] = manualDeal['dealType']
+                data[0]['dealPoints'] = manualDeal['dealPoints']
+                data[0]['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
+                toScrapMap[manualDeal['sku']] = data[0]
+        else:
+            data = toScrapMap.get(manualDeal['sku'])
+            data['dealFlag'] = 1
+            data['dealType'] = manualDeal['dealType']
+            data['dealPoints'] = manualDeal['dealPoints']
+            data['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
     pool = ThreadPool(cpu_count() *2)
     pool.map(scrapeFlipkart,toScrapMap.values())
     pool.close()
     pool.join()
     print "joining threads at %s"%(str(datetime.now()))
 def scrapeFlipkart(data):
     if data['source_id']!=2:
         return
     retryCount = 0
-    print str(data['identifier'])
     if data['identifier'] is None or len(data['identifier'].strip())==0:
         print "returning in valid identifier"
         return
     try:
         if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
-            print "sku id is already updated",data['_id']
+            print "sku id is already updated %d" %(data['_id'])
             return
     except:
         pass
     lowestSp = 0
     inStock = 0
+    buyBoxPrice = 0
+    isBuyBox = 0
     scraperFk = FlipkartScraper.FlipkartScraper()
     scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
     try:
         if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
             result = scraperProductPage.read(data['marketPlaceUrl'])
             if result.get('lowestSp')!=0:
                 lowestSp = result.get('lowestSp')
                 inStock = result.get('inStock')
+                buyBoxPrice = result.get('buyBoxPrice')
     except:
-        print "Unable to scrape product page ",data['identifier']
+        print "Unable to scrape product page %s" %(data['identifier'])
     if lowestSp == 0:
         url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
         while(retryCount < 3):
             try:
-                vendorsData = scraperFk.read(url)
+                vendorsData, buyBoxInfo = (scraperFk.read(url))
                 fetched = True
                 break
             except Exception as e:
                 print "***Retry count ",retryCount
                 retryCount+=1
                 if retryCount == 3:
                     fetched = False
                 print e
         if not fetched:
-            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
+            print "Unable to fetch data after multiple tries.Continue for %s"%(data['identifier'])
             return
         sortedVendorsData = []
         sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
         print "data",sortedVendorsData
             if iterator == 0:
                 lowestSp = vData['sellingPrice']
             break
         if lowestSp > 0:
             inStock = 1
+        if len(buyBoxInfo) > 0:
+            buyBoxPrice = buyBoxInfo[0].get('sellingPrice')
+        else:
+            print "No info about buy box for %d"%(data.get('_id'))
     print lowestSp
     print inStock
+    if buyBoxPrice is not None and buyBoxPrice == lowestSp:
+        isBuyBox = 1
     if lowestSp > 0:
-        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
+        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)
         get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
     else:
-        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
+        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':isBuyBox}}, multi=True)
         get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
     try:
-        recomputeDeal(data['skuBundleId'])
+        recomputeDeal(data)
     except:
-        print "Unable to compute deal for ",data['skuBundleId']
+        print "Unable to compute deal for %s"%(data['skuBundleId'])
+def recomputePoints(item, deal):
+    try:
+        nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
+    except:
+        print traceback.print_exc()
+        nlcPoints = deal['nlcPoints']
+    if item['manualDealThresholdPrice'] >= deal['available_price']:
+        dealPoints = item['dealPoints']
+    else:
+        dealPoints = 0
+    get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
 def populateNegativeDeals():
     negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
     mc.set("negative_deals", negativeDeals, 600)
-def recomputeDeal(skuBundleId):
+def recomputeDeal(item):
     """Lets recompute deal for this bundle"""
-    print "Recomputing for bundleId",skuBundleId
+    print "Recomputing for bundleId %d" %(item.get('skuBundleId'))
+    skuBundleId = item['skuBundleId']
     similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
     bestPrice = float("inf")
     bestOne = None
     bestSellerPoints = 0
     toUpdate = []
     for similarItem in similarItems:
         if mc.get("negative_deals") is None:
             populateNegativeDeals()
-        try:
+#        try:
-            cashBack = getCashBack(similarItem['_id'], similarItem['source_id'], similarItem['category_id'], mc, options.mongoHost)
+#            cashBack = getCashBack(similarItem['_id'], similarItem['source_id'], similarItem['category_id'], mc, options.mongoHost)
-            if not cashBack or cashBack.get('cash_back_status')!=1:
+#            if not cashBack or cashBack.get('cash_back_status')!=1:
-                pass
+#                pass
-            else:
+#            else:
-                if cashBack['cash_back_type'] ==1:
+#                if cashBack['cash_back_type'] ==1:
-                    similarItem['available_price'] = similarItem['available_price'] - similarItem['available_price'] * float(cashBack['cash_back'])/100
+#                    similarItem['available_price'] = similarItem['available_price'] - similarItem['available_price'] * float(cashBack['cash_back'])/100
-                elif cashBack['cash_back_type'] ==2:
+#                elif cashBack['cash_back_type'] ==2:
-                    similarItem['available_price'] = similarItem['available_price'] - float(cashBack['cash_back'])
+#                    similarItem['available_price'] = similarItem['available_price'] - float(cashBack['cash_back'])
-                else:
+#                else:
-                    pass
+#                    pass
-        except Exception as cashBackEx:
+#        except Exception as cashBackEx:
-            print cashBackEx
+#            print cashBackEx
-            print "Error calculating cashback."
+#            print "Error calculating cashback."
+        if similarItem['_id'] == item['_id']:
+            recomputePoints(item, similarItem)
         if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
             get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
             continue
         if similarItem['available_price'] < bestPrice:
             bestOne = similarItem

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/scripts/FlipkartPriceScraping.py – Rev 14705 → 15269