Subversion Repositories SmartDukaan

import urllib2

import simplejson as json

import pymongo

from dtr.utils.utils import to_java_date, getNlcPoints, transformUrl

from datetime import datetime, timedelta

from operator import itemgetter

from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper

from dtr.utils import AmazonDealScraper

from dtr.utils import FlipkartScraper,NewFlipkartScraper, ShopCluesScraper, \

PaytmOfferScraper, PaytmScraper, HomeShop18Scraper

from dtr.storage.MemCache import MemCache

from functools import partial

import threading

from dtr.utils.utils import getCashBack

import traceback

from shop2020.config.client.ConfigClient import ConfigClient

import chardet

from dtr.CouponMaster import addToPaytmMaster

config_client = ConfigClient()

host_memCache = config_client.get_property('mem_cache_host_dtr')

host = config_client.get_property('mongo_dtr_host')

saholicPricingHost = config_client.get_property('dtr_pricing_host')

mc = MemCache(host_memCache)

con  = None

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6,'HOMESHOP18':7}

headers = { 

           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'

        }

def get_mongo_connection(port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def calculateCashBack(data, inputPrice):

    try:

        cashBack = getCashBack(data['_id'], data['source_id'], data['category_id'], mc, 'localhost')

        print "CashBack is ",cashBack

        if not cashBack or cashBack.get('cash_back_status')!=1:

            data['cash_back_type'] = 0

            data['cash_back'] = 0

        else:

            if cashBack['cash_back_type'] in (1,2):

                if cashBack.get('maxCashBack') is not None:

                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*data['available_price'])/100 > cashBack.get('maxCashBack'):

                        cashBack['cash_back_type'] = 2

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    else:

                        pass

                data['cash_back_type'] = cashBack['cash_back_type']

                data['cash_back'] = float(cashBack['cash_back'])

            else:

                data['cash_back_type'] = 0

                data['cash_back'] = 0

    except Exception as cashBackEx:

        traceback.print_exc()

        print "Error calculating cashback."

        data['cash_back_type'] = 0

        data['cash_back'] = 0

    if data['cash_back_type'] == 1:

        data['netPriceAfterCashBack'] = inputPrice - float(inputPrice)*data['cash_back']/100

    elif data['cash_back_type'] == 2:

        data['netPriceAfterCashBack'] = inputPrice - data['cash_back']

    else:

        data['netPriceAfterCashBack'] = inputPrice

    return data

def returnLatestPrice(data, source_id, ignoreLastUpdated = True):

    now = datetime.now()

    if source_id == 1:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data['dealFlag'] ==1 and data['dealType'] ==1:

                data['marketPlaceUrl'] = data['dealUrl'].strip()

            if data.get('ignorePricing') ==1:

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            except:

                pass

            url = "http://www.amazon.in/gp/aw/ol/%s?o=New&op=1"%(data['identifier'])

            lowestPrice = 0.0

            try:

                if data['dealFlag'] ==1 and data['dealType'] ==1:

                    print "Inside deal"

                    deal_url = data['marketPlaceUrl']

                    print deal_url

                    dealScraperAmazon = AmazonDealScraper.AmazonScraper(True)

                    lowestPrice = dealScraperAmazon.read(deal_url)

                    print lowestPrice

                    if lowestPrice == 0:

                        raise

                else:    

                    scraperAmazon = AmazonScraper(True)

                    lowestPrice = scraperAmazon.read(url)

            except Exception as e:

                print e

                scraperAmazon = AmazonScraper(True)

                lowestPrice = scraperAmazon.read(url)

            print "LowestPrice ",lowestPrice

            inStock = 0

            if lowestPrice > 0:

                inStock = 1

            if lowestPrice > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id ==4:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data.get('ignorePricing') ==1:

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'], 'packQuantity':data['quantity']}

#             try:

#                 if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

#                     print "sku id is already updated",data['_id'] 

#                     return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

#             except:

#                 pass

#             

            url = "http://"+saholicPricingHost+":8080/mobileapi/dtr-pricing?id=%s"%(data['identifier'])

            lowestPrice = 0.0

            instock = 0

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            json_input = response.read()

            response.close()

            priceInfo = json.loads(json_input)

            print "PriceInfo ",data['identifier']

            print priceInfo

            lowestPrice = priceInfo['response']['sellingPrice']

            cheapestBulkPrice = None

            try:

                bulkPricing = priceInfo['response']['bulkPricing']

                for k,v in bulkPricing.iteritems():

                    if cheapestBulkPrice is None:

                        cheapestBulkPrice = v

                        continue

                    if cheapestBulkPrice > v:

                        cheapestBulkPrice = v

            except:

                cheapestBulkPrice = 0

            if lowestPrice > 0:

                instock = 1

            if instock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':instock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer'], 'cheapestBulkPrice': cheapestBulkPrice, 'packQuantity':data['quantity']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer'],'packQuantity':data['quantity']}

    elif source_id ==3:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}

            except Exception as e:

                print "Exception snapdeal"

                print e

            url="http://www.snapdeal.com/acors/json/v2/gvbps?supc=%s&catUrl=&bn=&catId=175&start=0&count=10000&scoreCategoryUrl=mobiles-mobile-phones&make2Order=false&auto=false&isO2OVendorRequired=true"%(data['identifier'])

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            snapdeal_data = response.read()

            encoding =  chardet.detect(snapdeal_data)

            try:

                snapdeal_data = snapdeal_data.decode(encoding.get('encoding'))

            except:

                snapdeal_data = snapdeal_data.decode(encoding.get('latin-1'))

            vendorInfo = json.loads(snapdeal_data)

            response.close()

            lowestOfferPrice = 0

            inStock = 0

            buyBoxPrice = 0

            isBuyBox = 1

            try:

                buyBoxStock = vendorInfo['primaryVendor']['buyableInventory']

                if buyBoxStock >0:

                    buyBoxPrice = vendorInfo['primaryVendor']['sellingPrice']

            except:

                pass

            sortedVendorsData = sorted(vendorInfo['vendors'], key=itemgetter('sellingPrice'))

            for sortedVendorData in sortedVendorsData:

                lowestOfferPrice = float(sortedVendorData['sellingPrice'])

                try:

                    stock = sortedVendorData['buyableInventory']

                except:

                    stock = 0

                if stock > 0 and lowestOfferPrice > 0:

                    inStock = 1

                    break

            print lowestOfferPrice

            print inStock

            print "*************"

            if buyBoxPrice != lowestOfferPrice:

                isBuyBox = 0

            if inStock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestOfferPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print traceback.print_exc()

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id == 2:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']} 

            except:

                pass

            lowestSp = 0

            inStock = 0

            buyBoxPrice = 0

            isBuyBox = 0

            scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()

            try:

                result = scraperProductPage.read(data['identifier'])

                print result

                if result.get('lowestSp')!=0:

                    lowestSp = result.get('lowestSp')

                    inStock = result.get('inStock')

                    buyBoxPrice = result.get('buyBoxPrice')

            except:

                print "Unable to scrape product page ",data['identifier']

            print lowestSp

            print inStock

            if buyBoxPrice is not None and buyBoxPrice == lowestSp:

                isBuyBox = 1

            if lowestSp > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestSp = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id == 5:

        try:

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            except:

                pass

            url = data['marketPlaceUrl']

            lowestPrice = 0.0

            try:

                sc = ShopCluesScraper.ShopCluesScraper()

                url = transformUrl(url, 5)

                productInfo = sc.read(url)

            except Exception as e:

                traceback.print_exc()

                productInfo = {}

                productInfo['price'] = 0.0

                productInfo['inStock'] = 0

                productInfo['isCod'] = 0

                productInfo['coupon'] = ""

            print "LowestPrice ",productInfo['price']

            if productInfo['price'] > 0 and productInfo['inStock']==1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'],'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'],'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':productInfo['inStock']}})

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'] , 'in_stock':productInfo['inStock'],'dealType':data['dealType'], 'rank':data['rank'],'codAvailable':productInfo['isCod']}})

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}})

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'rank':data['rank'],'codAvailable':productInfo['isCod']}})

            return {'_id':data['_id'],'available_price':productInfo['price'],'in_stock':productInfo['inStock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            traceback.print_exc()

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id == 6:

        try:

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}

            except:

                pass

            paytmScraper = PaytmScraper.PaytmScraper()

            url = "https://catalog.paytm.com/v1/mobile/product/%s"%(data['identifier'].strip())

            try:

                result = paytmScraper.read(url)

                print result

                effective_price = result.get('offerPrice')

                gross_price = effective_price

                shareUrl = result.get('shareUrl')

                if shareUrl is None:

                    shareUrl = data['marketPlaceUrl']

                coupon = ""

                try:

                    offers = PaytmOfferScraper.fetchOffers(result['offerUrl'])

                    try:

                        addToPaytmMaster(offers.get('codes'))

                    except:

                        print "Error in adding coupon"

                        traceback.print_exc()

                    bestOffer = {}

                    for offer_data in offers.get('codes'):

                        if effective_price > offer_data.get('effective_price'):

                            effective_price = offer_data.get('effective_price')

                            bestOffer = offer_data

                            coupon = bestOffer.get('code')

                except:

                    pass

                """Temp fix"""

                if len(coupon) > 0:

                    result['codAvailable'] = 0

                available_price = effective_price 

                if result['inStock']:

                    inStock = 1

                    get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':available_price,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':1,'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon,'gross_price':gross_price,'marketPlaceUrl':shareUrl}})

                    get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':available_price , 'in_stock':1,'codAvailable':result.get('codAvailable'),'gross_price':gross_price}})

                else:

                    inStock = 0

                    get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon,'marketPlaceUrl':shareUrl}})

                    get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'codAvailable':result.get('codAvailable')}})

            except:

                inStock = 0

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1}})

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})

            return {'_id':data['_id'],'available_price':available_price,'in_stock':inStock,'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':shareUrl,'thumbnail':data['thumbnail'], 'coupon':coupon,'codAvailable':result['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':gross_price}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            traceback.print_exc()

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}

    elif source_id ==7:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}

            except Exception as e:

                print "Exception snapdeal"

                print e

            response = None

            try:

                scraper = HomeShop18Scraper.HomeShop18Scraper()

                response = scraper.read('http://m.homeshop18.com/product.mobi?productId=%s'%(str(data['identifier'])))

            except Exception as e:

                print e

                scraper = HomeShop18Scraper.HomeShop18Scraper()

                response = scraper.read('http://m.homeshop18.com/product.mobi?productId=%s'%(str(data['identifier'])))

            lowestOfferPrice = 0

            inStock = 0

            if response is not None:

                lowestOfferPrice = float(response['price']+response['shippingCharge'])

                inStock = response['inStock']          

            print lowestOfferPrice

            print inStock

            print "*************"

            if lowestOfferPrice ==0:

                inStock = 0

            if inStock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestOfferPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print traceback.print_exc()

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    else:

        return {}

def recomputePoints(deal):

    try:

        nlcPoints = getNlcPoints(deal)

    except:

        traceback.print_exc()

        nlcPoints = deal['nlcPoints']

    bundleDealPoints = list(get_mongo_connection().Catalog.DealPoints.find({'skuBundleId':deal['skuBundleId'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))

    if len(bundleDealPoints) > 0:

        manualDealThresholdPrice = bundleDealPoints[0]['dealThresholdPrice']

        dealPoints = bundleDealPoints[0]['dealPoints']

    else:

        dealPoints = 0

        manualDealThresholdPrice = None

    if deal['available_price'] > manualDealThresholdPrice:

        dealPoints = 0

    if dealPoints != deal['dealPoints'] or manualDealThresholdPrice != deal['manualDealThresholdPrice'] or nlcPoints != deal['nlcPoints']:

        get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': manualDealThresholdPrice}})

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)

def recomputeDeal(item):

    """Lets recompute deal for this bundle"""

    print "Recomputing for bundleId",item.get('skuBundleId')

    skuBundleId = item['skuBundleId']

    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))

    bestPrice = float("inf")

    bestOne = None

    bestSellerPoints = 0

    toUpdate = []

    prepaidBestPrice = float("inf")

    prepaidBestOne = None

    prepaidBestSellerPoints = 0

    for similarItem in similarItems:

        try:

            recomputePoints(similarItem)

        except:

            traceback.print_exc()

        if similarItem['codAvailable'] ==1:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem['netPriceAfterCashBack'] < bestPrice:

                bestOne = similarItem

                bestPrice = similarItem['netPriceAfterCashBack']

                bestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem['netPriceAfterCashBack'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:

                bestOne = similarItem

                bestPrice = similarItem['netPriceAfterCashBack']

                bestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

        else:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem['netPriceAfterCashBack'] < prepaidBestPrice:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem['netPriceAfterCashBack']

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem['netPriceAfterCashBack'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem['netPriceAfterCashBack']

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

    #print "bestOne ", bestOne

    #print "prepaid best one",  prepaidBestOne

    if bestOne is not None or prepaidBestOne is not None:

        for similarItem in similarItems:

            toUpdate.append(similarItem['_id'])

        if bestOne is not None:

            toUpdate.remove(bestOne['_id'])

            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})

        if prepaidBestOne is not None:

            if bestOne is not None:

                if prepaidBestOne['netPriceAfterCashBack'] < bestOne['netPriceAfterCashBack']: 

                    toUpdate.remove(prepaidBestOne['_id'])

                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

            else:

                toUpdate.remove(prepaidBestOne['_id'])

                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

    if len(toUpdate) > 0:

        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

    print "Done with recomputing"

def getLatestPrice(skuBundleId, source_id):

    temp = []

    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))

    item = None

    for item in itemIds:

        item['dealFlag'] = 0

        item['dealType'] = 0

        item['dealUrl'] = ""

        if item['source_id'] ==5 and item['rank'] == 0 and item['category_id']!=6:

            continue

        if item['source_id'] ==3:

            item['marketPlaceUrl'] = item['marketPlaceUrl']+'?supc='+item.get('identifier')

        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))

        if len(manualDeals) > 0:

            item['dealFlag'] = 1

            item['dealType'] =manualDeals[0]['dealType']

            item['dealUrl'] = manualDeals[0]['dealUrl']

        info = returnLatestPrice(item, source_id)

        if item['source_id'] != SOURCE_MAP.get('PAYTM.COM'):

            data = calculateCashBack(item, info['available_price'])

        else:

            if info['codAvailable'] ==0:

                data = calculateCashBack(item, info['gross_price'])

            else:

                data = calculateCashBack(item, info['available_price'])  #No need, can be done in if with or clause.

        info['cash_back'] = data['cash_back']

        info['cash_back_type'] = data['cash_back_type']

        info['netPriceAfterCashBack'] = data['netPriceAfterCashBack']

        info['showNetPrice'] = data['showNetPrice']

        get_mongo_connection().Catalog.Deals.update({'_id':item['_id']},{"$set":{'netPriceAfterCashBack':info['netPriceAfterCashBack']}})

        temp.append(info)

    if item is not None:

        try:

            thread = threading.Thread(target=recomputeDeal, args = (item,))

            thread.daemon = True

            thread.start()    

        except:

            print traceback.print_exc()

            print "Unable to compute deal for ",skuBundleId

    return temp

def getLatestPriceById(id):

    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))

    item[0]['dealFlag'] = 0

    item[0]['dealType'] = 0

    item[0]['dealUrl'] = ""

    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item[0]['source_id'], 'sku':item[0]['_id']}))

    if len(manualDeals) > 0:

        item[0]['dealFlag'] = 1

        item[0]['dealType'] =manualDeals[0]['dealType']

        item[0]['dealUrl'] = manualDeals[0]['dealUrl']

    info = returnLatestPrice(item[0], item[0]['source_id'])

    if item[0]['source_id'] != SOURCE_MAP.get('PAYTM.COM'):

        data = calculateCashBack(item[0], info['available_price'])

    else:

        if info['codAvailable'] ==0:

            data = calculateCashBack(item[0], info['gross_price'])

        else:

            data = calculateCashBack(item[0], info['available_price'])  #No need, can be done in if with or clause.

    info['cash_back'] = data['cash_back']

    info['cash_back_type'] = data['cash_back_type']

    info['netPriceAfterCashBack'] = data['netPriceAfterCashBack']

    info['showNetPrice'] = data['showNetPrice']

    get_mongo_connection().Catalog.Deals.update({'_id':item[0]['_id']},{"$set":{'netPriceAfterCashBack':info['netPriceAfterCashBack']}})

    print info

    try:

        thread = threading.Thread(target=recomputeDeal, args = (item[0],))

        thread.daemon = True

        thread.start()    

    except:

        print "Unable to compute deal for ",item[0]['skuBundleId']

    return info

def updatePriceForNotificationBundles(skuBundleId):

    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'priceUpdatedOn':{'$lte':to_java_date(datetime.now() - timedelta(minutes=2))},'source_id':{"$in":SOURCE_MAP.values()}}))

    for item in itemIds:

        print item['_id']

        item['dealFlag'] = 0

        item['dealType'] = 0

        item['dealUrl'] = ""

        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item['source_id'], 'sku':item['_id']}))

        if len(manualDeals) > 0:

            item['dealFlag'] = 1

            item['dealType'] =manualDeals[0]['dealType']

            item['dealUrl'] = manualDeals[0]['dealUrl']

        info = returnLatestPrice(item, item['source_id'],False)

        print info

def main():

    print datetime.now()

    print "retuned %s"%(str(getLatestPrice(5899,4)))

    print datetime.now()

if __name__=='__main__':

    main()