WebSVN – SmartDukaan – Diff – /trunk/PyDTR/src/dtr/utils/FetchLivePrices.py

 import urllib2
 import simplejson as json
 import pymongo
-from dtr.utils.utils import to_java_date, getNlcPoints
+from dtr.utils.utils import to_java_date, getNlcPoints, changeToMobileUrl
 from datetime import datetime, timedelta
 from operator import itemgetter
 from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper
 from dtr.utils import AmazonDealScraper
-from dtr.utils import FlipkartScraper,NewFlipkartScraper
+from dtr.utils import FlipkartScraper,NewFlipkartScraper, ShopCluesScraper
 from dtr.storage.MemCache import MemCache
 from functools import partial
 import threading
 from dtr.utils.utils import getCashBack
 import traceback
 host = config_client.get_property('mongo_dtr_host')
 mc = MemCache(host_memCache)
-con = None
+con  = None
-SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4}
+SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
 headers = {
            'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
             'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
             'Accept-Language' : 'en-US,en;q=0.8',
             'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
+        }
-ignoreItems = [23219]
+ignoreItems = []
 def get_mongo_connection(port=27017):
     global con
     if con is None:
         print "Establishing connection %s host and port %d" %(host,port)
         except Exception as e:
             print "Exception for _id %d and source %s"%(data['_id'], source_id)
             print e
             return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
+    elif source_id == 5:
+        try:
+            if data['_id'] in ignoreItems:
+                print "Ignored items returning for %d"%(data['_id'])
+                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
+            try:
+                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
+                    print "sku id is already updated",data['_id']
+                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable']}
+            except:
+                pass
+            url = data['marketPlaceUrl']
+            lowestPrice = 0.0
+            try:
+                sc = ShopCluesScraper.ShopCluesScraper()
+                url = changeToMobileUrl(url, 5)
+                productInfo = sc.read(url)
+            except Exception as e:
+                raise
+            print "LowestPrice ",productInfo['price']
+            if productInfo['price'] > 0 and productInfo['inStock']==1:
+                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'],'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'],'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':productInfo['inStock']}})
+                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'] , 'in_stock':productInfo['inStock'],'dealType':data['dealType'], 'rank':data['rank']}})
+            else:
+                lowestPrice = data['available_price']
+                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}})
+                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'rank':data['rank']}})
+            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':productInfo['inStock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod']}
+        except Exception as e:
+            print "Exception for _id %d and source %s"%(data['_id'], source_id)
+            print e
+            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable']}
     else:
         return {}
 def recomputePoints(item, deal):
     try:
         nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
     except:
         if mc.get("negative_deals") is None:
             populateNegativeDeals()
         if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
             get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
             continue
+        if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
+            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
+            continue
         if similarItem['available_price'] < bestPrice:
             bestOne = similarItem
             bestPrice = similarItem['available_price']
             bestSellerPoints = similarItem['bestSellerPoints']
         elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
 def getLatestPrice(skuBundleId, source_id):
     temp = []
     itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))
     for item in itemIds:
+        if item['source_id'] ==5 and item['rank'] == 0:
+            continue
         item['dealFlag'] = 0
         item['dealType'] = 0
         item['dealPoints'] = 0
         item['manualDealThresholdPrice'] = None
         if item['source_id'] ==3:
             item['manualDealThresholdPrice'] = manualDeals[0]['dealThresholdPrice']
         info = returnLatestPrice(item, item['source_id'],False)
         print info
 def main():
-    print "retuned %s"%(str(getLatestPriceById(8840)))
+    print "retuned %s"%(str(getLatestPriceById(23827)))
 if __name__=='__main__':
     main()

Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/utils/FetchLivePrices.py – Rev 15855 → 15902