Subversion Repositories SmartDukaan

import urllib2
import simplejson as json
import pymongo
from dtr.utils.utils import to_java_date, getNlcPoints, transformUrl, DEAL_PRIORITY
from datetime import datetime, timedelta
from operator import itemgetter
from shop2020.model.v1.catalog.script import AmazonAsyncScraper
from dtr.utils import FlipkartScraper,NewFlipkartScraper, ShopCluesScraper, \
PaytmOfferScraper, PaytmScraper, HomeShop18Scraper
from dtr.storage.MemCache import MemCache
from functools import partial
import threading
from dtr.utils.utils import getCashBack
import traceback
from shop2020.config.client.ConfigClient import ConfigClient
import chardet
from dtr.CouponMaster import addToPaytmMaster
from math import floor

config_client = ConfigClient()
host_memCache = config_client.get_property('mem_cache_host_dtr')
host = config_client.get_property('mongo_dtr_host')

saholicPricingHost = config_client.get_property('dtr_pricing_host')

mc = MemCache(host_memCache)


con  = None
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6,'HOMESHOP18':7}

headers = { 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      
            'Accept-Language' : 'en-US,en;q=0.8',                     
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
        }

amScraper = AmazonAsyncScraper.Products("AKIAII3SGRXBJDPCHSGQ", "B92xTbNBTYygbGs98w01nFQUhbec1pNCkCsKVfpg", "AF6E3O0VE0X4D")
marketplaceId = 'A21TJRUUN4KGV'

def get_mongo_connection(port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def calculateCashBack(data, inputPrice):
    try:
        cashBack = getCashBack(data['_id'], data['source_id'], data['category_id'], mc, 'localhost')
        print "CashBack is ",cashBack
        if not cashBack or cashBack.get('cash_back_status')!=1:
            data['cash_back_type'] = 0
            data['cash_back'] = 0
        else:
            if cashBack['cash_back_type'] in (1,2):
                
                if cashBack.get('maxCashBack') is not None:
                    
                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*data['available_price'])/100 > cashBack.get('maxCashBack'):
                        cashBack['cash_back_type'] = 2
                        cashBack['cash_back'] = cashBack['maxCashBack']
                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):
                        cashBack['cash_back'] = cashBack['maxCashBack']
                    else:
                        pass
                
                data['cash_back_type'] = cashBack['cash_back_type']
                data['cash_back'] = float(cashBack['cash_back'])
            else:
                data['cash_back_type'] = 0
                data['cash_back'] = 0
    except Exception as cashBackEx:
        traceback.print_exc()
        print "Error calculating cashback."
        data['cash_back_type'] = 0
        data['cash_back'] = 0
        
    if data['cash_back_type'] == 1:
        data['netPriceAfterCashBack'] = inputPrice - floor(float(inputPrice)*data['cash_back']/100)
    elif data['cash_back_type'] == 2:
        data['netPriceAfterCashBack'] = inputPrice - data['cash_back']
    else:
        data['netPriceAfterCashBack'] = inputPrice
    
    return data

def returnLatestPrice(data, source_id, ignoreLastUpdated = True):
    now = datetime.now()
    if source_id == 1:
        try:
            if data['identifier'] is None or len(data['identifier'].strip()) !=10:
                return {}
            
            if data['dealFlag'] ==1 and data['dealType'] ==1:
                data['marketPlaceUrl'] = data['dealUrl'].strip()
            
            
            if data.get('ignorePricing') ==1:
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            except:
                pass
            
            
            lowestPrice = 0.0
            asinPricingMap = amScraper.get_competitive_pricing_for_asin(marketplaceId, [data['identifier'].strip().upper()])
            print "asinPricingMap ",asinPricingMap
            lowestPrice = asinPricingMap.get(data['identifier'].strip().upper()) 
            print "LowestPrice ",lowestPrice
            inStock = 0
            if lowestPrice > 0:
                inStock = 1
            if lowestPrice > 0:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)
            
            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        
    elif source_id ==4:
        
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'], 'packQuantity':data['quantity']}
            
#             try:
#                 if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
#                     print "sku id is already updated",data['_id'] 
#                     return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
#             except:
#                 pass
#             
            url = "http://"+saholicPricingHost+":8080/mobileapi/dtr-pricing?id=%s"%(data['identifier'])
            lowestPrice = 0.0
            instock = 0
            req = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(req)
            json_input = response.read()
            response.close()
            priceInfo = json.loads(json_input)
            print "PriceInfo ",data['identifier']
            print priceInfo
            lowestPrice = priceInfo['response']['sellingPrice']
            cheapestBulkPrice = None
            try:
                bulkPricing = priceInfo['response']['bulkPricing']
                for k,v in bulkPricing.iteritems():
                    if cheapestBulkPrice is None:
                        cheapestBulkPrice = v
                        continue
                    if cheapestBulkPrice > v:
                        cheapestBulkPrice = v
            except:
                cheapestBulkPrice = 0
            
            if lowestPrice > 0:
                instock = 1
            if instock  == 1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':instock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)
            
            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer'], 'cheapestBulkPrice': cheapestBulkPrice, 'packQuantity':data['quantity']}
        
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer'],'packQuantity':data['quantity']}
        
        
    elif source_id ==3:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id']
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}
                
            except Exception as e:
                print "Exception snapdeal"
                print e
            
            url="https://m.snapdeal.com/snap/product/getRefreshProductDetails?supc=%s"%(data['identifier'])
            req = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(req)
            snapdeal_data = response.read()
            encoding =  chardet.detect(snapdeal_data)
            try:
                snapdeal_data = snapdeal_data.decode(encoding.get('encoding'))
            except:
                snapdeal_data = snapdeal_data.decode(encoding.get('latin-1'))
            vendor_data = json.loads(snapdeal_data)
            response.close()
            lowestOfferPrice = 0
            inStock = 0
            buyBoxPrice = 0
            isBuyBox = 1
            try:
                buyBoxStock = int(vendor_data['vendorDtlSRO']['vendorDetailInventoryPricingSRO']['buyableInventory'])
                soldOut = vendor_data['vendorDtlSRO']['vendorDetailInventoryPricingSRO']['soldOut']
                inStock = 1 if not soldOut else 0
                if buyBoxStock >0:
                    buyBoxPrice = float(vendor_data['vendorDtlSRO']['finalPrice'])
                    
                lowestOfferPrice = float(vendor_data['auxiliarySellerInfo']['priceStartRange'])
            except:
                pass
                    
            print lowestOfferPrice
            print inStock
            print "*************"
            
            if buyBoxPrice != lowestOfferPrice:
                isBuyBox = 0
            
            if inStock  == 1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestOfferPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            
            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        except Exception as e:
            print traceback.print_exc()
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        
    elif source_id == 2:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id']
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']} 
            except:
                pass
            
            lowestSp = 0
            inStock = 0
            buyBoxPrice = 0
            isBuyBox = 0
            scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
            try:
                result = scraperProductPage.read(data['identifier'])
                print result
                if result.get('lowestSp')!=0:
                    lowestSp = result.get('lowestSp')
                    inStock = result.get('inStock')
                    buyBoxPrice = result.get('buyBoxPrice')
            except:
                print "Unable to scrape product page ",data['identifier']
            print lowestSp
            print inStock
            if buyBoxPrice is not None and buyBoxPrice == lowestSp:
                isBuyBox = 1
            if lowestSp > 0:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestSp = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
                
            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        
    elif source_id == 5:
        try:
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            except:
                pass
            
            
            url = data['marketPlaceUrl']
            lowestPrice = 0.0
            try:
                sc = ShopCluesScraper.ShopCluesScraper()
                url = transformUrl(url, 5)
                productInfo = sc.read(url)
            except Exception as e:
                traceback.print_exc()
                productInfo = {}
                productInfo['price'] = 0.0
                productInfo['inStock'] = 0
                productInfo['isCod'] = 0
                productInfo['coupon'] = ""
                
            print "LowestPrice ",productInfo['price']
            if productInfo['price'] > 0 and productInfo['inStock']==1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'],'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'],'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':productInfo['inStock']}})
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'] , 'in_stock':productInfo['inStock'],'dealType':data['dealType'], 'rank':data['rank'],'codAvailable':productInfo['isCod']}})
            else:
                lowestPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}})
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'rank':data['rank'],'codAvailable':productInfo['isCod']}})
            
            return {'_id':data['_id'],'available_price':productInfo['price'],'in_stock':productInfo['inStock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'], 'tagline': data['tagline'], 'offer': data['offer']}
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            traceback.print_exc()
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
    
    elif source_id == 6:
        try:
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}
            except:
                pass
            
            paytmScraper = PaytmScraper.PaytmScraper()
            url = "https://catalog.paytm.com/v1/mobile/product/%s"%(data['identifier'].strip())
            try:
                result = paytmScraper.read(url)
                print result
                effective_price = result.get('offerPrice')
                gross_price = effective_price
                shareUrl = result.get('shareUrl')
                if shareUrl is None:
                    shareUrl = data['marketPlaceUrl']
                coupon = ""
                try:
                    offers = PaytmOfferScraper.fetchOffers(result['offerUrl'])
                    bestOffer = {}
                    for offer_data in offers.get('codes'):
                        if effective_price > offer_data.get('effective_price'):
                            effective_price = offer_data.get('effective_price')
                            bestOffer = offer_data
                            coupon = bestOffer.get('code')
                except:
                    pass
                
                """Temp fix"""
                if len(coupon) > 0:
                    result['codAvailable'] = 0
                
                available_price = effective_price 
                if result['inStock']:
                    inStock = 1
                    get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':available_price,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':1,'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon,'gross_price':gross_price,'marketPlaceUrl':shareUrl}})
                    get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':available_price , 'in_stock':1,'codAvailable':result.get('codAvailable'),'gross_price':gross_price}})
                else:
                    inStock = 0
                    get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon,'marketPlaceUrl':shareUrl}})
                    get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'codAvailable':result.get('codAvailable')}})
                
            except:
                inStock = 0
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1}})
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})
                
                    
            return {'_id':data['_id'],'available_price':available_price,'in_stock':inStock,'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':shareUrl,'thumbnail':data['thumbnail'], 'coupon':coupon,'codAvailable':result['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':gross_price}
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            traceback.print_exc()
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}
    
    elif source_id ==7:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id']
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}
                
            except Exception as e:
                print "Exception snapdeal"
                print e
            
            response = None
            
            try:
                scraper = HomeShop18Scraper.HomeShop18Scraper()
                response = scraper.read('http://m.homeshop18.com/product.mobi?productId=%s'%(str(data['identifier'])))
            except Exception as e:
                print e
                scraper = HomeShop18Scraper.HomeShop18Scraper()
                response = scraper.read('http://m.homeshop18.com/product.mobi?productId=%s'%(str(data['identifier'])))
                
            lowestOfferPrice = 0
            inStock = 0
            
            if response is not None:
                lowestOfferPrice = float(response['price']+response['shippingCharge'])
                inStock = response['inStock']          
                    
            print lowestOfferPrice
            print inStock
            print "*************"
            
            if lowestOfferPrice ==0:
                inStock = 0
            
            
            if inStock  == 1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestOfferPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            
            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        except Exception as e:
            print traceback.print_exc()
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        
    else:
        return {}
    
    

def recomputePoints(deal):
    try:
        nlcPoints = getNlcPoints(deal)
    except:
        traceback.print_exc()
        nlcPoints = deal['nlcPoints']
    
    bundleDealPoints = list(get_mongo_connection().Catalog.DealPoints.find({'skuBundleId':deal['skuBundleId'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))
    if len(bundleDealPoints) > 0:
        manualDealThresholdPrice = bundleDealPoints[0]['dealThresholdPrice']
        dealPoints = bundleDealPoints[0]['dealPoints']
    else:
        dealPoints = 0
        manualDealThresholdPrice = None
    
    if deal['available_price'] > manualDealThresholdPrice:
        dealPoints = 0
         
        
    if dealPoints != deal['dealPoints'] or manualDealThresholdPrice != deal['manualDealThresholdPrice'] or nlcPoints != deal['nlcPoints']:
        get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': manualDealThresholdPrice}})


def populateNegativeDeals():
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
    mc.set("negative_deals", negativeDeals, 600)

def recomputeDeal(item):
    
    """Lets recompute deal for this bundle"""
    print "Recomputing for bundleId",item.get('skuBundleId')
    skuBundleId = item['skuBundleId']
    
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))
    bestPrice = float("inf")
    bestOne = None
    toUpdate = []
    prepaidBestPrice = float("inf")
    prepaidBestOne = None
    for similarItem in similarItems:
        try:
            recomputePoints(similarItem)
        except:
            traceback.print_exc()
        if similarItem['codAvailable'] ==1:
            if mc.get("negative_deals") is None:
                populateNegativeDeals()
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                continue
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                continue
            if similarItem['netPriceAfterCashBack'] < bestPrice:
                bestOne = similarItem
                bestPrice = similarItem['netPriceAfterCashBack']
            elif similarItem['netPriceAfterCashBack'] == bestPrice:
                
                try:
                    if (DEAL_PRIORITY.index(int(similarItem['source_id'])) > DEAL_PRIORITY.index(int(bestOne['source_id']))):
                        continue
                except:
                    traceback.print_exc()
                
                bestOne = similarItem
                bestPrice = similarItem['netPriceAfterCashBack']
            else:
                pass
        else:
            if mc.get("negative_deals") is None:
                populateNegativeDeals()
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                continue
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                continue
            if similarItem['netPriceAfterCashBack'] < prepaidBestPrice:
                prepaidBestOne = similarItem
                prepaidBestPrice = similarItem['netPriceAfterCashBack']
            elif similarItem['netPriceAfterCashBack'] == prepaidBestPrice:
                
                try:
                    if (DEAL_PRIORITY.index(int(similarItem['source_id'])) > DEAL_PRIORITY.index(int(prepaidBestOne['source_id']))):
                        continue
                except:
                    traceback.print_exc()
                
                prepaidBestOne = similarItem
                prepaidBestPrice = similarItem['netPriceAfterCashBack']
            else:
                pass
    if bestOne is not None or prepaidBestOne is not None:
        for similarItem in similarItems:
            toUpdate.append(similarItem['_id'])
        if bestOne is not None:
            toUpdate.remove(bestOne['_id'])
            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
        if prepaidBestOne is not None:
            if bestOne is not None:
                if prepaidBestOne['netPriceAfterCashBack'] < bestOne['netPriceAfterCashBack']: 
                    toUpdate.remove(prepaidBestOne['_id'])
                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
            else:
                toUpdate.remove(prepaidBestOne['_id'])
                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
    if len(toUpdate) > 0:
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

    print "Done with recomputing"

def getLatestPrice(skuBundleId, source_id):
    temp = []
    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))
    item = None
    for item in itemIds:
        item['dealFlag'] = 0
        item['dealType'] = 0
        item['dealUrl'] = ""
        if item['source_id'] ==5 and item['rank'] == 0 and item['category_id']!=6:
            continue
        if item['source_id'] ==3:
            item['marketPlaceUrl'] = item['marketPlaceUrl']+'?supc='+item.get('identifier')
        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))
        if len(manualDeals) > 0:
            item['dealFlag'] = 1
            item['dealType'] =manualDeals[0]['dealType']
            item['dealUrl'] = manualDeals[0]['dealUrl']
        info = returnLatestPrice(item, source_id)
        if item['source_id'] != SOURCE_MAP.get('PAYTM.COM'):
            data = calculateCashBack(item, info['available_price'])
        else:
            if info['codAvailable'] ==0:
                data = calculateCashBack(item, info['gross_price'])
            else:
                data = calculateCashBack(item, info['available_price'])  #No need, can be done in if with or clause.
        
        info['cash_back'] = data['cash_back']
        info['cash_back_type'] = data['cash_back_type']
        info['netPriceAfterCashBack'] = data['netPriceAfterCashBack']
        info['showNetPrice'] = data['showNetPrice']
        info['category_id'] = item['category_id']
        info['subCategoryId'] = item['subCategoryId']
        info['identifier'] = str(item['identifier'])
        get_mongo_connection().Catalog.Deals.update({'_id':item['_id']},{"$set":{'netPriceAfterCashBack':info['netPriceAfterCashBack']}})
        temp.append(info)
    if item is not None:
        try:
            thread = threading.Thread(target=recomputeDeal, args = (item,))
            thread.daemon = True
            thread.start()    
        except:
            print traceback.print_exc()
            print "Unable to compute deal for ",skuBundleId
    return temp

def getLatestPriceById(id):
    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))
    item[0]['dealFlag'] = 0
    item[0]['dealType'] = 0
    item[0]['dealUrl'] = ""
    
    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item[0]['source_id'], 'sku':item[0]['_id']}))
    if len(manualDeals) > 0:
        item[0]['dealFlag'] = 1
        item[0]['dealType'] =manualDeals[0]['dealType']
        item[0]['dealUrl'] = manualDeals[0]['dealUrl']
    
    info = returnLatestPrice(item[0], item[0]['source_id'])
    if item[0]['source_id'] != SOURCE_MAP.get('PAYTM.COM'):
        data = calculateCashBack(item[0], info['available_price'])
    else:
        if info['codAvailable'] ==0:
            data = calculateCashBack(item[0], info['gross_price'])
        else:
            data = calculateCashBack(item[0], info['available_price'])  #No need, can be done in if with or clause.
    
    info['cash_back'] = data['cash_back']
    info['cash_back_type'] = data['cash_back_type']
    info['netPriceAfterCashBack'] = data['netPriceAfterCashBack']
    info['showNetPrice'] = data['showNetPrice']
    get_mongo_connection().Catalog.Deals.update({'_id':item[0]['_id']},{"$set":{'netPriceAfterCashBack':info['netPriceAfterCashBack']}})
    print info
    try:
        thread = threading.Thread(target=recomputeDeal, args = (item[0],))
        thread.daemon = True
        thread.start()    
    except:
        print "Unable to compute deal for ",item[0]['skuBundleId']
    return info

def updatePriceForNotificationBundles(skuBundleId):
    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'priceUpdatedOn':{'$lte':to_java_date(datetime.now() - timedelta(minutes=2))},'source_id':{"$in":SOURCE_MAP.values()}}))
    for item in itemIds:
        print item['_id']
        item['dealFlag'] = 0
        item['dealType'] = 0
        item['dealUrl'] = ""
        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item['source_id'], 'sku':item['_id']}))
        if len(manualDeals) > 0:
            item['dealFlag'] = 1
            item['dealType'] =manualDeals[0]['dealType']
            item['dealUrl'] = manualDeals[0]['dealUrl']
        info = returnLatestPrice(item, item['source_id'],False)
        print info

def main():
    print datetime.now()
    print "returned %s"%(str(getLatestPriceById(23014)))
    print datetime.now()
if __name__=='__main__':
    main()