WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/dtr/utils/FetchLivePrices.py

 import urllib2
 import simplejson as json
 import pymongo
 from dtr.utils.utils import to_java_date
-from datetime import datetime
+from datetime import datetime, timedelta
 from operator import itemgetter
 from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper
 from dtr.utils import FlipkartScraper
 con = None
-now = datetime.now()
 SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3}
 scraperFk = FlipkartScraper.FlipkartScraper()
 scraperAmazon = AmazonScraper()
 headers = {
             print e
             return None
     return con
 def returnLatestPrice(data, source_id):
+    now = datetime.now()
     if source_id == 1:
         try:
-            if data['identifier'] is None or len(data['identifier'])==0:
+            if data['identifier'] is None or len(data['identifier'].strip())==0:
                 return {}
+            try:
+                if data['updatedOn'] + timedelta(minutes=5) > to_java_date(now):
+                    print "sku id is already updated",data['_id']
+                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
+            except:
+                pass
             url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data['identifier'])
             lowestPrice = 0.0
             lowestPrice = scraperAmazon.read(url)
             inStock = 0
             if lowestPrice > 0:
                 inStock = 1
             if lowestPrice > 0:
-                get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier'],'source_id':source_id}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock}}, multi=True)
             else:
-                get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier'],'source_id':source_id}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0}}, multi=True)
+                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0}}, multi=True)
+                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}}, multi=True)
+            try:
+                recomputeDeal(data['skuBundleId'])
+            except:
+                print "Unable to compute deal for ",data['skuBundleId']
             return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
         except:
             return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
     elif source_id ==3:
         try:
-            if data['identifier'] is None or len(data['identifier'])==0:
+            if data['identifier'] is None or len(data['identifier'].strip())==0:
                 return {}
+            try:
+                if data['updatedOn'] + timedelta(minutes=5) > to_java_date(now):
+                    print "sku id is already updated",data['_id']
+                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
+            except:
+                pass
             url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(data['identifier'])
             req = urllib2.Request(url,headers=headers)
             response = urllib2.urlopen(req)
             json_input = response.read()
             vendorInfo = json.loads(json_input)
                     inStock = 1
                     break
             print lowestOfferPrice
             print inStock
-            print stock
             print "*************"
             if inStock  == 1:
-                get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier'],'source_id':source_id}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock}}, multi=True)
             else:
-                get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier'],'source_id':source_id}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
+            try:
+                recomputeDeal(data['skuBundleId'])
+            except:
+                print "Unable to compute deal for ",data['skuBundleId']
             return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
         except:
             return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
     elif source_id == 2:
         try:
-            if data['identifier'] is None or len(data['identifier'])==0:
+            if data['identifier'] is None or len(data['identifier'].strip())==0:
                 return {}
+            try:
+                if data['updatedOn'] + timedelta(minutes=5) > to_java_date(now):
+                    print "sku id is already updated",data['_id']
+                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
+            except:
+                pass
             url = "http://www.flipkart.com/ps/%s"%(data['identifier'])
             vendorsData = scraperFk.read(url)
             sortedVendorsData = []
             sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
             print "data",sortedVendorsData
-            lowestSp, iterator = (0,)*2
+            lowestSp = 0
+            inStock = 0
             for vData in sortedVendorsData:
-                if iterator == 0:
-                    lowestSp = vData['sellingPrice']
+                lowestSp = vData['sellingPrice']
                 break
             if lowestSp > 0:
                 inStock = 1
             print lowestSp
             print inStock
             if lowestSp > 0:
-                get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier'],'source_id':2}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
             else:
-                get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier'],'source_id':2}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
+                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
+            try:
+                recomputeDeal(data['skuBundleId'])
+            except:
+                print "Unable to compute deal for ",data['skuBundleId']
             return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
         except:
             return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
     else:
         return {}
+def recomputeDeal(skuBundleId):
+    """Lets recompute deal for this bundle"""
+    print "Recomputing for bundleId",skuBundleId
+    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
+    bestPrice = float("inf")
+    bestOne = None
+    bestSellerPoints = 0
+    toUpdate = []
+    for similarItem in similarItems:
+        if similarItem['in_stock'] == 0:
+            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
+            continue
+        if similarItem['available_price'] < bestPrice:
+            bestOne = similarItem
+            bestPrice = similarItem['available_price']
+            bestSellerPoints = similarItem['bestSellerPoints']
+        elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
+            bestOne = similarItem
+            bestPrice = similarItem['available_price']
+            bestSellerPoints = similarItem['bestSellerPoints']
+        else:
+            pass
+    if bestOne is not None:
+        for similarItem in similarItems:
+            toUpdate.append(similarItem['_id'])
+        toUpdate.remove(bestOne['_id'])
+        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
+    if len(toUpdate) > 0:
+        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
 def getLatestPrice(skuBundleId, source_id):
     temp = []
     itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))
     for item in itemIds:
     return temp
 def getLatestPriceById(id):
     item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))
     return returnLatestPrice(item[0], item[0]['source_id'])
 def main():
     print getLatestPriceById(7)

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/utils/FetchLivePrices.py – Rev 13866 → 13918