Subversion Repositories SmartDukaan

import urllib2

import simplejson as json

import pymongo

from dtr.utils.utils import to_java_date, getNlcPoints, transformUrl

from datetime import datetime, timedelta

from operator import itemgetter

from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper

from dtr.utils import AmazonDealScraper

from dtr.utils import FlipkartScraper,NewFlipkartScraper, ShopCluesScraper

from dtr.storage.MemCache import MemCache

from functools import partial

import threading

from dtr.utils.utils import getCashBack

import traceback

from shop2020.config.client.ConfigClient import ConfigClient

import chardet

config_client = ConfigClient()

host_memCache = config_client.get_property('mem_cache_host_dtr')

host = config_client.get_property('mongo_dtr_host')

mc = MemCache(host_memCache)

con  = None

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}

headers = { 

           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'

        }

ignoreItems = [24105, 23177]

def get_mongo_connection(port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def returnLatestPrice(data, source_id, ignoreLastUpdated = True):

    now = datetime.now()

    if source_id == 1:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data['_id'] in ignoreItems:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            if data['dealFlag'] ==1 and data['dealType'] ==1:

                data['marketPlaceUrl'] = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            except:

                pass

            url = "http://www.amazon.in/gp/aw/ol/%s?o=New&op=1"%(data['identifier'])

            lowestPrice = 0.0

            try:

                if data['dealFlag'] ==1 and data['dealType'] ==1:

                    print "Inside deal"

                    deal_url = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())

                    print deal_url

                    dealScraperAmazon = AmazonDealScraper.AmazonScraper(True)

                    lowestPrice = dealScraperAmazon.read(deal_url)

                    print lowestPrice

                    if lowestPrice == 0:

                        raise

                else:    

                    scraperAmazon = AmazonScraper(True)

                    lowestPrice = scraperAmazon.read(url)

            except Exception as e:

                print e

                scraperAmazon = AmazonScraper(True)

                lowestPrice = scraperAmazon.read(url)

            print "LowestPrice ",lowestPrice

            inStock = 0

            if lowestPrice > 0:

                inStock = 1

            if lowestPrice > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id ==4:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            except:

                pass

            url = "http://50.116.3.101:8080/mobileapi/dtr-pricing?id=%s"%(data['identifier'])

            lowestPrice = 0.0

            instock = 0

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            json_input = response.read()

            response.close()

            priceInfo = json.loads(json_input)

            lowestPrice = priceInfo['response']['sellingPrice']

            if lowestPrice > 0:

                instock = 1

            if instock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':instock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id ==3:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data['_id'] in ignoreItems:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}

            except Exception as e:

                print "Exception snapdeal"

                print e

            url="http://www.snapdeal.com/acors/json/v2/gvbps?supc=%s&catUrl=&bn=&catId=175&start=0&count=10000"%(data['identifier'])

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            snapdeal_data = response.read()

            encoding =  chardet.detect(snapdeal_data)

            try:

                snapdeal_data = snapdeal_data.decode(encoding.get('encoding'))

            except:

                snapdeal_data = snapdeal_data.decode(encoding.get('latin-1'))

            vendorInfo = json.loads(snapdeal_data)

            response.close()

            lowestOfferPrice = 0

            inStock = 0

            buyBoxPrice = 0

            isBuyBox = 1

            try:

                buyBoxStock = vendorInfo['primaryVendor']['buyableInventory']

                if buyBoxStock >0:

                    buyBoxPrice = vendorInfo['primaryVendor']['sellingPrice']

            except:

                pass

            print "buy box stock ",buyBoxStock

            print "bux box price ",buyBoxPrice

            sortedVendorsData = sorted(vendorInfo['vendors'], key=itemgetter('sellingPrice'))

            for sortedVendorData in sortedVendorsData:

                lowestOfferPrice = float(sortedVendorData['sellingPrice'])

                try:

                    stock = sortedVendorData['buyableInventory']

                except:

                    stock = 0

                if stock > 0 and lowestOfferPrice > 0:

                    inStock = 1

                    break

            print lowestOfferPrice

            print inStock

            print "*************"

            if buyBoxPrice != lowestOfferPrice:

                isBuyBox = 0

            if inStock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestOfferPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print traceback.print_exc()

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id == 2:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data['_id'] in ignoreItems:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']} 

            except:

                pass

            lowestSp = 0

            inStock = 0

            buyBoxPrice = 0

            isBuyBox = 0

            scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()

            try:

                if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):

                    result = scraperProductPage.read(data['marketPlaceUrl'])

                    print result

                    if result.get('lowestSp')!=0:

                        lowestSp = result.get('lowestSp')

                        inStock = result.get('inStock')

                        buyBoxPrice = result.get('buyBoxPrice')

            except:

                print "Unable to scrape product page ",data['identifier']

            if lowestSp ==0:

                url = "http://www.flipkart.com/ps/%s"%(data['identifier'])

                scraperFk = FlipkartScraper.FlipkartScraper()

                vendorsData, buyBoxInfo = (scraperFk.read(url))

                sortedVendorsData = []

                sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))

                print "data",sortedVendorsData

                for vData in sortedVendorsData:

                    lowestSp = vData['sellingPrice']

                    break

                if lowestSp > 0:

                    inStock = 1

                try:

                    buyBoxPrice = buyBoxInfo[0].get('sellingPrice')

                except:

                    buyBoxPrice = None

            print lowestSp

            print inStock

            if buyBoxPrice is not None and buyBoxPrice == lowestSp:

                isBuyBox = 1

            if lowestSp > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestSp = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id == 5:

        try:

            if data['_id'] in ignoreItems:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            except:

                pass

            url = data['marketPlaceUrl']

            lowestPrice = 0.0

            try:

                sc = ShopCluesScraper.ShopCluesScraper()

                url = transformUrl(url, 5)

                productInfo = sc.read(url)

            except Exception as e:

                traceback.print_exc()

                productInfo = {}

                productInfo['price'] = 0.0

                productInfo['inStock'] = 0

                productInfo['isCod'] = 0

                productInfo['coupon'] = ""

            print "LowestPrice ",productInfo['price']

            if productInfo['price'] > 0 and productInfo['inStock']==1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'],'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'],'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':productInfo['inStock']}})

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'] , 'in_stock':productInfo['inStock'],'dealType':data['dealType'], 'rank':data['rank'],'codAvailable':productInfo['isCod']}})

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}})

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'rank':data['rank'],'codAvailable':productInfo['isCod']}})

            return {'_id':data['_id'],'available_price':productInfo['price'],'in_stock':productInfo['inStock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            traceback.print_exc()

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    else:

        return {}

def recomputePoints(item, deal):

    try:

        nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])

    except:

        print traceback.print_exc()

        nlcPoints = deal['nlcPoints']

    if item['manualDealThresholdPrice'] >= deal['available_price']:

        dealPoints = item['dealPoints']

    else:

        dealPoints = 0

    get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)

def recomputeDeal(item):

    """Lets recompute deal for this bundle"""

    print "Recomputing for bundleId",item.get('skuBundleId')

    skuBundleId = item['skuBundleId']

    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))

    bestPrice = float("inf")

    bestOne = None

    bestSellerPoints = 0

    toUpdate = []

    prepaidBestPrice = float("inf")

    prepaidBestOne = None

    prepaidBestSellerPoints = 0

    for similarItem in similarItems:

        if similarItem['_id'] == item['_id']:

            recomputePoints(item, similarItem)

        if similarItem['codAvailable'] ==1:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem['available_price'] < bestPrice:

                bestOne = similarItem

                bestPrice = similarItem['available_price']

                bestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:

                bestOne = similarItem

                bestPrice = similarItem['available_price']

                bestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

        else:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem['available_price'] < prepaidBestPrice:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem['available_price']

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem['available_price']

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

    print "bestOne ", bestOne

    print "prepaid best one",  prepaidBestOne

    if bestOne is not None or prepaidBestOne is not None:

        for similarItem in similarItems:

            toUpdate.append(similarItem['_id'])

        if bestOne is not None:

            toUpdate.remove(bestOne['_id'])

            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})

        if prepaidBestOne is not None:

            if bestOne is not None:

                if prepaidBestOne['available_price'] < bestOne['available_price']: 

                    toUpdate.remove(prepaidBestOne['_id'])

                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

            else:

                toUpdate.remove(prepaidBestOne['_id'])

                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

    if len(toUpdate) > 0:

        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

    print "Done with recomputing"

def getLatestPrice(skuBundleId, source_id):

    temp = []

    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))

    item = None

    for item in itemIds:

        item['dealFlag'] = 0

        item['dealType'] = 0

        item['dealPoints'] = 0

        item['manualDealThresholdPrice'] = None

        if item['source_id'] ==5 and item['rank'] == 0:

            continue

        if item['source_id'] ==3:

            item['marketPlaceUrl'] = item['marketPlaceUrl']+'?supc='+item.get('identifier')

        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))

        if len(manualDeals) > 0:

            item['dealFlag'] = 1

            item['dealType'] =manualDeals[0]['dealType']

            item['dealPoints'] = manualDeals[0]['dealPoints']

            item['manualDealThresholdPrice'] = manualDeals[0]['dealThresholdPrice']

        info = returnLatestPrice(item, source_id)

        print info

        try:

            cashBack = getCashBack(item['_id'], item['source_id'], item['category_id'], mc, 'localhost')

            print "CashBack is ",cashBack

            if not cashBack or cashBack.get('cash_back_status')!=1:

                info['cash_back_type'] = 0

                info['cash_back'] = 0

            else:

                if cashBack['cash_back_type'] in (1,2):

                    info['cash_back_type'] = cashBack['cash_back_type']

                    info['cash_back'] = float(cashBack['cash_back'])

                else:

                    info['cash_back_type'] = 0

                    info['cash_back'] = 0

        except Exception as cashBackEx:

            traceback.print_exc()

            print "Error calculating cashback."

            info['cash_back_type'] = 0

            info['cash_back'] = 0

        print "info is ",info

        temp.append(info)

    if item is not None:

        try:

            thread = threading.Thread(target=recomputeDeal, args = (item,))

            thread.daemon = True

            thread.start()    

        except:

            print traceback.print_exc()

            print "Unable to compute deal for ",skuBundleId

    return temp

def getLatestPriceById(id):

    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))

    item[0]['dealFlag'] = 0

    item[0]['dealType'] = 0

    item[0]['dealPoints'] = 0

    item[0]['manualDealThresholdPrice'] = None

    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item[0]['source_id'], 'sku':item[0]['_id']}))

    if len(manualDeals) > 0:

        item[0]['dealFlag'] = 1

        item[0]['dealType'] =manualDeals[0]['dealType']

        item[0]['dealPoints'] = manualDeals[0]['dealPoints']

        item[0]['manualDealThresholdPrice'] = manualDeals[0]['dealThresholdPrice']

    info = returnLatestPrice(item[0], item[0]['source_id'])

    print info

    try:

        cashBack = getCashBack(item[0]['_id'], item[0]['source_id'], item[0]['category_id'], mc, 'localhost')

        print "CashBack is ",cashBack

        if not cashBack or cashBack.get('cash_back_status')!=1:

            info['cash_back_type'] = 0

            info['cash_back'] = 0

        else:

            if cashBack['cash_back_type'] in (1,2):

                info['cash_back_type'] = cashBack['cash_back_type']

                info['cash_back'] = float(cashBack['cash_back'])

            else:

                info['cash_back_type'] = 0

                info['cash_back'] = 0

    except Exception as cashBackEx:

        traceback.print_exc()

        print cashBackEx

        print "Error calculating cashback."

        info['cash_back_type'] = 0

        info['cash_back'] = 0

    try:

        thread = threading.Thread(target=recomputeDeal, args = (item[0],))

        thread.daemon = True

        thread.start()    

    except:

        print "Unable to compute deal for ",item[0]['skuBundleId']

    return info

def updatePriceForNotificationBundles(skuBundleId):

    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'priceUpdatedOn':{'$lte':to_java_date(datetime.now() - timedelta(minutes=2))},'source_id':{"$in":SOURCE_MAP.values()}}))

    for item in itemIds:

        print item['_id']

        item['dealFlag'] = 0

        item['dealType'] = 0

        item['dealPoints'] = 0

        item['manualDealThresholdPrice'] = None

        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item['source_id'], 'sku':item['_id']}))

        if len(manualDeals) > 0:

            item['dealFlag'] = 1

            item['dealType'] =manualDeals[0]['dealType']

            item['dealPoints'] = manualDeals[0]['dealPoints']

            item['manualDealThresholdPrice'] = manualDeals[0]['dealThresholdPrice']

        info = returnLatestPrice(item, item['source_id'],False)

        print info

def main():

    print datetime.now()

    print "retuned %s"%(str(getLatestPriceById(4490)))

    print datetime.now()

if __name__=='__main__':

    main()