Subversion Repositories SmartDukaan

import urllib2
import simplejson as json
import pymongo
from dtr.utils.utils import to_java_date, getNlcPoints, transformUrl
from datetime import datetime, timedelta
from operator import itemgetter
from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper
from dtr.utils import AmazonDealScraper
from dtr.utils import FlipkartScraper,NewFlipkartScraper, ShopCluesScraper, \
PaytmOfferScraper, PaytmScraper, HomeShop18Scraper
from dtr.storage.MemCache import MemCache
from functools import partial
import threading
from dtr.utils.utils import getCashBack
import traceback
from shop2020.config.client.ConfigClient import ConfigClient
import chardet
from dtr.CouponMaster import addToPaytmMaster

config_client = ConfigClient()
host_memCache = config_client.get_property('mem_cache_host_dtr')
host = config_client.get_property('mongo_dtr_host')

mc = MemCache(host_memCache)


con  = None
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6,'HOMESHOP18':7}

headers = { 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      
            'Accept-Language' : 'en-US,en;q=0.8',                     
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
        }


def get_mongo_connection(port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def returnLatestPrice(data, source_id, ignoreLastUpdated = True):
    now = datetime.now()
    if source_id == 1:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                if data['dealFlag'] ==1 and data['dealType'] ==1:
                    data['marketPlaceUrl'] = data['dealUrl'].strip()
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            
            if data['dealFlag'] ==1 and data['dealType'] ==1:
                data['marketPlaceUrl'] = data['dealUrl'].strip()
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            except:
                pass
            
            
            url = "http://www.amazon.in/gp/aw/ol/%s?o=New&op=1"%(data['identifier'])
            lowestPrice = 0.0
            try:
                if data['dealFlag'] ==1 and data['dealType'] ==1:
                    print "Inside deal"
                    deal_url = data['marketPlaceUrl']
                    print deal_url
                    dealScraperAmazon = AmazonDealScraper.AmazonScraper(True)
                    lowestPrice = dealScraperAmazon.read(deal_url)
                    print lowestPrice
                    if lowestPrice == 0:
                        raise
                else:    
                    scraperAmazon = AmazonScraper(True)
                    lowestPrice = scraperAmazon.read(url)
            except Exception as e:
                print e
                scraperAmazon = AmazonScraper(True)
                lowestPrice = scraperAmazon.read(url)
            print "LowestPrice ",lowestPrice
            inStock = 0
            if lowestPrice > 0:
                inStock = 1
            if lowestPrice > 0:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)
            
            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        
    elif source_id ==4:
        
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            except:
                pass
            
            url = "http://50.116.3.101:8080/mobileapi/dtr-pricing?id=%s"%(data['identifier'])
            lowestPrice = 0.0
            instock = 0
            req = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(req)
            json_input = response.read()
            response.close()
            priceInfo = json.loads(json_input)
            lowestPrice = priceInfo['response']['sellingPrice']
            if lowestPrice > 0:
                instock = 1
            if instock  == 1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':instock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)
            
            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}
        
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}
        
        
    elif source_id ==3:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id']
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}
                
            except Exception as e:
                print "Exception snapdeal"
                print e
            
            url="http://www.snapdeal.com/acors/json/v2/gvbps?supc=%s&catUrl=&bn=&catId=175&start=0&count=10000&scoreCategoryUrl=mobiles-mobile-phones&make2Order=false&auto=false&isO2OVendorRequired=true"%(data['identifier'])
            req = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(req)
            snapdeal_data = response.read()
            encoding =  chardet.detect(snapdeal_data)
            try:
                snapdeal_data = snapdeal_data.decode(encoding.get('encoding'))
            except:
                snapdeal_data = snapdeal_data.decode(encoding.get('latin-1'))
            vendorInfo = json.loads(snapdeal_data)
            response.close()
            lowestOfferPrice = 0
            inStock = 0
            buyBoxPrice = 0
            isBuyBox = 1
            try:
                buyBoxStock = vendorInfo['primaryVendor']['buyableInventory']
                if buyBoxStock >0:
                    buyBoxPrice = vendorInfo['primaryVendor']['sellingPrice']
            except:
                pass
            print "buy box stock ",buyBoxStock
            print "bux box price ",buyBoxPrice
            sortedVendorsData = sorted(vendorInfo['vendors'], key=itemgetter('sellingPrice'))
            for sortedVendorData in sortedVendorsData:
                lowestOfferPrice = float(sortedVendorData['sellingPrice'])
                try:
                    stock = sortedVendorData['buyableInventory']
                except:
                    stock = 0
                if stock > 0 and lowestOfferPrice > 0:
                    inStock = 1
                    break
                    
            print lowestOfferPrice
            print inStock
            print "*************"
            
            if buyBoxPrice != lowestOfferPrice:
                isBuyBox = 0
            
            if inStock  == 1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestOfferPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            
            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        except Exception as e:
            print traceback.print_exc()
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        
    elif source_id == 2:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id']
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']} 
            except:
                pass
            
            lowestSp = 0
            inStock = 0
            buyBoxPrice = 0
            isBuyBox = 0
            scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
            try:
                result = scraperProductPage.read(data['identifier'])
                print result
                if result.get('lowestSp')!=0:
                    lowestSp = result.get('lowestSp')
                    inStock = result.get('inStock')
                    buyBoxPrice = result.get('buyBoxPrice')
            except:
                print "Unable to scrape product page ",data['identifier']
            print lowestSp
            print inStock
            if buyBoxPrice is not None and buyBoxPrice == lowestSp:
                isBuyBox = 1
            if lowestSp > 0:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestSp = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
                
            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        
    elif source_id == 5:
        try:
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            except:
                pass
            
            
            url = data['marketPlaceUrl']
            lowestPrice = 0.0
            try:
                sc = ShopCluesScraper.ShopCluesScraper()
                url = transformUrl(url, 5)
                productInfo = sc.read(url)
            except Exception as e:
                traceback.print_exc()
                productInfo = {}
                productInfo['price'] = 0.0
                productInfo['inStock'] = 0
                productInfo['isCod'] = 0
                productInfo['coupon'] = ""
                
            print "LowestPrice ",productInfo['price']
            if productInfo['price'] > 0 and productInfo['inStock']==1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'],'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'],'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':productInfo['inStock']}})
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'] , 'in_stock':productInfo['inStock'],'dealType':data['dealType'], 'rank':data['rank'],'codAvailable':productInfo['isCod']}})
            else:
                lowestPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}})
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'rank':data['rank'],'codAvailable':productInfo['isCod']}})
            
            return {'_id':data['_id'],'available_price':productInfo['price'],'in_stock':productInfo['inStock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'], 'tagline': data['tagline'], 'offer': data['offer']}
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            traceback.print_exc()
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
    
    elif source_id == 6:
        try:
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}
            except:
                pass
            
            paytmScraper = PaytmScraper.PaytmScraper()
            url = "https://catalog.paytm.com/v1/mobile/product/%s"%(data['identifier'].strip())
            try:
                result = paytmScraper.read(url)
                effective_price = result.get('offerPrice')
                gross_price = effective_price
                coupon = ""
                try:
                    offers = PaytmOfferScraper.fetchOffers(result['offerUrl'])
                    try:
                        addToPaytmMaster(offers.get('codes'))
                    except:
                        print "Error in adding coupon"
                        traceback.print_exc()
                    bestOffer = {}
                    for offer_data in offers.get('codes'):
                        if effective_price > offer_data.get('effective_price'):
                            effective_price = offer_data.get('effective_price')
                            bestOffer = offer_data
                            coupon = bestOffer.get('code')
                except:
                    pass
                
                """Temp fix"""
                if len(coupon) > 0:
                    result['codAvailable'] = 0
                
                available_price = effective_price 
                if result['inStock']:
                    inStock = 1
                    get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':available_price,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':1,'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon,'gross_price':gross_price}})
                    get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':available_price , 'in_stock':1,'codAvailable':result.get('codAvailable'),'gross_price':gross_price}})
                else:
                    inStock = 0
                    get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon}})
                    get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'codAvailable':result.get('codAvailable')}})
                
            except:
                inStock = 0
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1}})
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})
                
                    
            return {'_id':data['_id'],'available_price':available_price,'in_stock':inStock,'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':coupon,'codAvailable':result['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':gross_price}
        except Exception as e:
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            traceback.print_exc()
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}
    
    elif source_id ==7:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data.get('ignorePricing') ==1:
                print "Ignored items returning for %d"%(data['_id'])
                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
            
            try:
                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id']
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}
                
            except Exception as e:
                print "Exception snapdeal"
                print e
            
            response = None
            
            try:
                scraper = HomeShop18Scraper.HomeShop18Scraper()
                response = scraper.read('http://m.homeshop18.com/product.mobi?productId=%s'%(str(data['identifier'])))
            except Exception as e:
                print e
                scraper = HomeShop18Scraper.HomeShop18Scraper()
                response = scraper.read('http://m.homeshop18.com/product.mobi?productId=%s'%(str(data['identifier'])))
                
            lowestOfferPrice = 0
            inStock = 0
            
            if response is not None:
                lowestOfferPrice = float(response['price']+response['shippingCharge'])
                inStock = response['inStock']          
                    
            print lowestOfferPrice
            print inStock
            print "*************"
            
            if lowestOfferPrice ==0:
                inStock = 0
            
            
            if inStock  == 1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            else:
                lowestOfferPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
            
            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        except Exception as e:
            print traceback.print_exc()
            print "Exception for _id %d and source %s"%(data['_id'], source_id)
            print e
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
        
    else:
        return {}
    
    

def recomputePoints(deal):
    try:
        nlcPoints = getNlcPoints(deal)
    except:
        traceback.print_exc()
        nlcPoints = deal['nlcPoints']
    
    bundleDealPoints = list(get_mongo_connection().Catalog.DealPoints.find({'skuBundleId':deal['skuBundleId'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))
    if len(bundleDealPoints) > 0:
        manualDealThresholdPrice = bundleDealPoints[0]['dealThresholdPrice']
        dealPoints = bundleDealPoints[0]['dealPoints']
    else:
        dealPoints = 0
        manualDealThresholdPrice = None
    
    if deal['available_price'] > manualDealThresholdPrice:
        dealPoints = 0
         
        
    if dealPoints != deal['dealPoints'] or manualDealThresholdPrice != deal['manualDealThresholdPrice'] or nlcPoints != deal['nlcPoints']:
        get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': manualDealThresholdPrice}})


def populateNegativeDeals():
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
    mc.set("negative_deals", negativeDeals, 600)

def recomputeDeal(item):
    
    """Lets recompute deal for this bundle"""
    print "Recomputing for bundleId",item.get('skuBundleId')
    skuBundleId = item['skuBundleId']
    
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
    bestPrice = float("inf")
    bestOne = None
    bestSellerPoints = 0
    toUpdate = []
    prepaidBestPrice = float("inf")
    prepaidBestOne = None
    prepaidBestSellerPoints = 0
    for similarItem in similarItems:
        try:
            recomputePoints(similarItem)
        except:
            traceback.print_exc()
        if similarItem['codAvailable'] ==1:
            if mc.get("negative_deals") is None:
                populateNegativeDeals()
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                continue
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                continue
            if similarItem['available_price'] < bestPrice:
                bestOne = similarItem
                bestPrice = similarItem['available_price']
                bestSellerPoints = similarItem['bestSellerPoints']
            elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
                bestOne = similarItem
                bestPrice = similarItem['available_price']
                bestSellerPoints = similarItem['bestSellerPoints']
            else:
                pass
        else:
            if mc.get("negative_deals") is None:
                populateNegativeDeals()
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                continue
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                continue
            if similarItem['source_id'] == SOURCE_MAP.get('PAYTM.COM'):
                similarItem['available_price'] = similarItem['gross_price']
            if similarItem['available_price'] < prepaidBestPrice:
                prepaidBestOne = similarItem
                prepaidBestPrice = similarItem['available_price']
                prepaidBestSellerPoints = similarItem['bestSellerPoints']
            elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
                prepaidBestOne = similarItem
                prepaidBestPrice = similarItem['available_price']
                prepaidBestSellerPoints = similarItem['bestSellerPoints']
            else:
                pass
    print "bestOne ", bestOne
    print "prepaid best one",  prepaidBestOne
    if bestOne is not None or prepaidBestOne is not None:
        for similarItem in similarItems:
            toUpdate.append(similarItem['_id'])
        if bestOne is not None:
            toUpdate.remove(bestOne['_id'])
            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
        if prepaidBestOne is not None:
            if bestOne is not None:
                if prepaidBestOne['available_price'] < bestOne['available_price']: 
                    toUpdate.remove(prepaidBestOne['_id'])
                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
            else:
                toUpdate.remove(prepaidBestOne['_id'])
                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
    if len(toUpdate) > 0:
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

    print "Done with recomputing"

def getLatestPrice(skuBundleId, source_id):
    temp = []
    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))
    item = None
    for item in itemIds:
        item['dealFlag'] = 0
        item['dealType'] = 0
        item['dealUrl'] = ""
        if item['source_id'] ==5 and item['rank'] == 0:
            continue
        if item['source_id'] ==3:
            item['marketPlaceUrl'] = item['marketPlaceUrl']+'?supc='+item.get('identifier')
        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))
        if len(manualDeals) > 0:
            item['dealFlag'] = 1
            item['dealType'] =manualDeals[0]['dealType']
            item['dealUrl'] = manualDeals[0]['dealUrl']
        info = returnLatestPrice(item, source_id)
        print info
        try:
            cashBack = getCashBack(item['_id'], item['source_id'], item['category_id'], mc, 'localhost')
            print "CashBack is ",cashBack
            if not cashBack or cashBack.get('cash_back_status')!=1:
                info['cash_back_type'] = 0
                info['cash_back'] = 0
            else:
                if cashBack['cash_back_type'] in (1,2):
                    
                    if cashBack.get('maxCashBack') is not None:
                        
                        if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*info['available_price'])/100 > cashBack.get('maxCashBack'):
                            cashBack['cash_back_type'] = 2
                            cashBack['cash_back'] = cashBack['maxCashBack']
                        elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):
                            cashBack['cash_back'] = cashBack['maxCashBack']
                        else:
                            pass
                    
                    
                    
                    info['cash_back_type'] = cashBack['cash_back_type']
                    info['cash_back'] = float(cashBack['cash_back'])
                else:
                    info['cash_back_type'] = 0
                    info['cash_back'] = 0
        except Exception as cashBackEx:
            traceback.print_exc()
            print "Error calculating cashback."
            info['cash_back_type'] = 0
            info['cash_back'] = 0
        print "info is ",info
        temp.append(info)
    if item is not None:
        try:
            thread = threading.Thread(target=recomputeDeal, args = (item,))
            thread.daemon = True
            thread.start()    
        except:
            print traceback.print_exc()
            print "Unable to compute deal for ",skuBundleId
    return temp

def getLatestPriceById(id):
    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))
    item[0]['dealFlag'] = 0
    item[0]['dealType'] = 0
    item[0]['dealUrl'] = ""
    
    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item[0]['source_id'], 'sku':item[0]['_id']}))
    if len(manualDeals) > 0:
        item[0]['dealFlag'] = 1
        item[0]['dealType'] =manualDeals[0]['dealType']
        item[0]['dealUrl'] = manualDeals[0]['dealUrl']
    
    info = returnLatestPrice(item[0], item[0]['source_id'])
    print info
    try:
        cashBack = getCashBack(item[0]['_id'], item[0]['source_id'], item[0]['category_id'], mc, 'localhost')
        print "CashBack is ",cashBack
        if not cashBack or cashBack.get('cash_back_status')!=1:
            info['cash_back_type'] = 0
            info['cash_back'] = 0
        else:
            if cashBack['cash_back_type'] in (1,2):
                info['cash_back_type'] = cashBack['cash_back_type']
                info['cash_back'] = float(cashBack['cash_back'])
            else:
                info['cash_back_type'] = 0
                info['cash_back'] = 0
    except Exception as cashBackEx:
        traceback.print_exc()
        print cashBackEx
        print "Error calculating cashback."
        info['cash_back_type'] = 0
        info['cash_back'] = 0
    try:
        thread = threading.Thread(target=recomputeDeal, args = (item[0],))
        thread.daemon = True
        thread.start()    
    except:
        print "Unable to compute deal for ",item[0]['skuBundleId']
    return info

def updatePriceForNotificationBundles(skuBundleId):
    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'priceUpdatedOn':{'$lte':to_java_date(datetime.now() - timedelta(minutes=2))},'source_id':{"$in":SOURCE_MAP.values()}}))
    for item in itemIds:
        print item['_id']
        item['dealFlag'] = 0
        item['dealType'] = 0
        item['dealUrl'] = ""
        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item['source_id'], 'sku':item['_id']}))
        if len(manualDeals) > 0:
            item['dealFlag'] = 1
            item['dealType'] =manualDeals[0]['dealType']
            item['dealUrl'] = manualDeals[0]['dealUrl']
        info = returnLatestPrice(item, item['source_id'],False)
        print info

def main():
    print datetime.now()
    print "retuned %s"%(str(getLatestPriceById(7)))
    print datetime.now()
if __name__=='__main__':
    main()