WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/scripts/HomeShop18PriceScraping.py

 from operator import itemgetter
 import chardet
 from dtr.utils import HomeShop18Scraper
+SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6, 'HOMESHOP18.COM':7}
 con = None
 parser = optparse.OptionParser()
 parser.add_option("-m", "--m", dest="mongoHost",
                       default="localhost",
 (options, args) = parser.parse_args()
 mc = MemCache(options.mongoHost)
+def getNetPriceForItem(itemId, source_id, category_id ,price):
+    cash_back_type = 0
+    cash_back = 0
+    try:
+        cashBack = getCashBack(itemId, source_id, category_id, mc, options.mongoHost)
+        if not cashBack or cashBack.get('cash_back_status')!=1:
+            cash_back_type = 0
+            cash_back = 0
+        else:
+            if cashBack['cash_back_type'] in (1,2):
+                if cashBack.get('maxCashBack') is not None:
+                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*price)/100 > cashBack.get('maxCashBack'):
+                        cashBack['cash_back_type'] = 2
+                        cashBack['cash_back'] = cashBack['maxCashBack']
+                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):
+                        cashBack['cash_back'] = cashBack['maxCashBack']
+                    else:
+                        pass
+                cash_back_type = cashBack['cash_back_type']
+                cash_back = float(cashBack['cash_back'])
+    except Exception as cashBackEx:
+        pass
+    if cash_back_type ==1:
+        return (price - float(cash_back)*price/100)
+    elif cash_back_type ==2:
+        return (price - cash_back)
+    else:
+        return price
 def populate():
     toScrapMap = {}
     bestSellers = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'rank':{'$gt':0}}))
     for bestSeller in bestSellers:
         snapdealBestSellers = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':7}))
     if lowestOfferPrice ==0:
         inStock = 0
     if inStock  == 1:
+        netPriceAfterCashBack = getNetPriceForItem(data['_id'], SOURCE_MAP.get('HOMESHOP18.COM'), data['category_id'], lowestOfferPrice)
         get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
-        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
+        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable'],'netPriceAfterCashBack':netPriceAfterCashBack}}, multi=True)
     else:
         lowestOfferPrice = data['available_price']
+        netPriceAfterCashBack = getNetPriceForItem(data['_id'], SOURCE_MAP.get('HOMESHOP18.COM'), data['category_id'], lowestOfferPrice)
         get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
-        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
+        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable'],'netPriceAfterCashBack':netPriceAfterCashBack}}, multi=True)
     try:
         recomputeDeal(data)
     except:
         print "Unable to compute deal for ",data['skuBundleId']
 def recomputeDeal(item):
     """Lets recompute deal for this bundle"""
     print "Recomputing for bundleId %d" %(item.get('skuBundleId'))
     skuBundleId = item['skuBundleId']
-    similarItems = list(get_mongo_connection(host=options.mongoHost).Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
+    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))
     bestPrice = float("inf")
     bestOne = None
     bestSellerPoints = 0
     toUpdate = []
     prepaidBestPrice = float("inf")
     for similarItem in similarItems:
         if similarItem['codAvailable'] ==1:
             if mc.get("negative_deals") is None:
                 populateNegativeDeals()
             if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
-                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
+                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                 continue
             if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
-                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
+                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                 continue
-            if similarItem['available_price'] < bestPrice:
+            if similarItem.get('netPriceAfterCashBack') < bestPrice:
                 bestOne = similarItem
-                bestPrice = similarItem['available_price']
+                bestPrice = similarItem.get('netPriceAfterCashBack')
                 bestSellerPoints = similarItem['bestSellerPoints']
-            elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
+            elif similarItem.get('netPriceAfterCashBack') == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
                 bestOne = similarItem
-                bestPrice = similarItem['available_price']
+                bestPrice = similarItem.get('netPriceAfterCashBack')
                 bestSellerPoints = similarItem['bestSellerPoints']
             else:
                 pass
         else:
             if mc.get("negative_deals") is None:
                 populateNegativeDeals()
             if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
-                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
+                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                 continue
             if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
-                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
+                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                 continue
-            if similarItem['source_id'] == SOURCE_MAP.get('PAYTM.COM'):
-                similarItem['available_price'] = similarItem['gross_price']
-            if similarItem['available_price'] < prepaidBestPrice:
+            if similarItem.get('netPriceAfterCashBack') < prepaidBestPrice:
                 prepaidBestOne = similarItem
-                prepaidBestPrice = similarItem['available_price']
+                prepaidBestPrice = similarItem.get('netPriceAfterCashBack')
                 prepaidBestSellerPoints = similarItem['bestSellerPoints']
-            elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
+            elif similarItem.get('netPriceAfterCashBack') == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
                 prepaidBestOne = similarItem
-                prepaidBestPrice = similarItem['available_price']
+                prepaidBestPrice = similarItem.get('netPriceAfterCashBack')
                 prepaidBestSellerPoints = similarItem['bestSellerPoints']
             else:
                 pass
     if bestOne is not None or prepaidBestOne is not None:
         for similarItem in similarItems:
             toUpdate.append(similarItem['_id'])
         if bestOne is not None:
             toUpdate.remove(bestOne['_id'])
-            get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
+            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
         if prepaidBestOne is not None:
             if bestOne is not None:
-                if prepaidBestOne['available_price'] < bestOne['available_price']:
+                if prepaidBestOne.get('netPriceAfterCashBack') < bestOne.get('netPriceAfterCashBack'):
                     toUpdate.remove(prepaidBestOne['_id'])
-                    get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
+                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
             else:
                 toUpdate.remove(prepaidBestOne['_id'])
-                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
+                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
     if len(toUpdate) > 0:
-        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
+        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
 def main():
     populate()

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/scripts/HomeShop18PriceScraping.py – Rev 17107 → 19191