Subversion Repositories SmartDukaan

import urllib2

import simplejson as json

import pymongo

from dtr.utils.utils import to_java_date, getNlcPoints, transformUrl

from datetime import datetime, timedelta

from operator import itemgetter

from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper

from dtr.utils import AmazonDealScraper

from dtr.utils import FlipkartScraper,NewFlipkartScraper, ShopCluesScraper, \

PaytmOfferScraper, PaytmScraper, HomeShop18Scraper

from dtr.storage.MemCache import MemCache

from functools import partial

import threading

from dtr.utils.utils import getCashBack

import traceback

from shop2020.config.client.ConfigClient import ConfigClient

import chardet

from dtr.CouponMaster import addToPaytmMaster

from math import floor

config_client = ConfigClient()

host_memCache = config_client.get_property('mem_cache_host_dtr')

host = config_client.get_property('mongo_dtr_host')

saholicPricingHost = config_client.get_property('dtr_pricing_host')

mc = MemCache(host_memCache)

con  = None

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6,'HOMESHOP18':7}

headers = { 

           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'

        }

def get_mongo_connection(port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def calculateCashBack(data, inputPrice):

    try:

        cashBack = getCashBack(data['_id'], data['source_id'], data['category_id'], mc, 'localhost')

        print "CashBack is ",cashBack

        if not cashBack or cashBack.get('cash_back_status')!=1:

            data['cash_back_type'] = 0

            data['cash_back'] = 0

        else:

            if cashBack['cash_back_type'] in (1,2):

                if cashBack.get('maxCashBack') is not None:

                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*data['available_price'])/100 > cashBack.get('maxCashBack'):

                        cashBack['cash_back_type'] = 2

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    else:

                        pass

                data['cash_back_type'] = cashBack['cash_back_type']

                data['cash_back'] = float(cashBack['cash_back'])

            else:

                data['cash_back_type'] = 0

                data['cash_back'] = 0

    except Exception as cashBackEx:

        traceback.print_exc()

        print "Error calculating cashback."

        data['cash_back_type'] = 0

        data['cash_back'] = 0

    if data['cash_back_type'] == 1:

        data['netPriceAfterCashBack'] = inputPrice - floor(float(inputPrice)*data['cash_back']/100)

    elif data['cash_back_type'] == 2:

        data['netPriceAfterCashBack'] = inputPrice - data['cash_back']

    else:

        data['netPriceAfterCashBack'] = inputPrice

    return data

def returnLatestPrice(data, source_id, ignoreLastUpdated = True):

    now = datetime.now()

    if source_id == 1:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data['dealFlag'] ==1 and data['dealType'] ==1:

                data['marketPlaceUrl'] = data['dealUrl'].strip()

            if data.get('ignorePricing') ==1:

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            except:

                pass

            url = "http://www.amazon.in/gp/aw/ol/%s?o=New&op=1"%(data['identifier'])

            lowestPrice = 0.0

            try:

                if data['dealFlag'] ==1 and data['dealType'] ==1:

                    print "Inside deal"

                    deal_url = data['marketPlaceUrl']

                    print deal_url

                    dealScraperAmazon = AmazonDealScraper.AmazonScraper(True)

                    lowestPrice = dealScraperAmazon.read(deal_url)

                    print lowestPrice

                    if lowestPrice == 0:

                        raise

                else:    

                    scraperAmazon = AmazonScraper(True)

                    lowestPrice = scraperAmazon.read(url)

            except Exception as e:

                print e

                scraperAmazon = AmazonScraper(True)

                lowestPrice = scraperAmazon.read(url)

            print "LowestPrice ",lowestPrice

            inStock = 0

            if lowestPrice > 0:

                inStock = 1

            if lowestPrice > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id ==4:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data.get('ignorePricing') ==1:

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'], 'packQuantity':data['quantity']}

#             try:

#                 if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

#                     print "sku id is already updated",data['_id'] 

#                     return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

#             except:

#                 pass

#             

            url = "http://"+saholicPricingHost+":8080/mobileapi/dtr-pricing?id=%s"%(data['identifier'])

            lowestPrice = 0.0

            instock = 0

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            json_input = response.read()

            response.close()

            priceInfo = json.loads(json_input)

            print "PriceInfo ",data['identifier']

            print priceInfo

            lowestPrice = priceInfo['response']['sellingPrice']

            cheapestBulkPrice = None

            try:

                bulkPricing = priceInfo['response']['bulkPricing']

                for k,v in bulkPricing.iteritems():

                    if cheapestBulkPrice is None:

                        cheapestBulkPrice = v

                        continue

                    if cheapestBulkPrice > v:

                        cheapestBulkPrice = v

            except:

                cheapestBulkPrice = 0

            if lowestPrice > 0:

                instock = 1

            if instock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':instock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer'], 'cheapestBulkPrice': cheapestBulkPrice, 'packQuantity':data['quantity']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer'],'packQuantity':data['quantity']}

    elif source_id ==3:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}

            except Exception as e:

                print "Exception snapdeal"

                print e

            url="http://www.snapdeal.com/acors/json/v2/gvbps?supc=%s&catUrl=&bn=&catId=175&start=0&count=10000&scoreCategoryUrl=mobiles-mobile-phones&make2Order=false&auto=false&isO2OVendorRequired=true"%(data['identifier'])

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            snapdeal_data = response.read()

            encoding =  chardet.detect(snapdeal_data)

            try:

                snapdeal_data = snapdeal_data.decode(encoding.get('encoding'))

            except:

                snapdeal_data = snapdeal_data.decode(encoding.get('latin-1'))

            vendorInfo = json.loads(snapdeal_data)

            response.close()

            lowestOfferPrice = 0

            inStock = 0

            buyBoxPrice = 0

            isBuyBox = 1

            try:

                buyBoxStock = vendorInfo['primaryVendor']['buyableInventory']

                if buyBoxStock >0:

                    buyBoxPrice = vendorInfo['primaryVendor']['sellingPrice']

            except:

                pass

            sortedVendorsData = sorted(vendorInfo['vendors'], key=itemgetter('sellingPrice'))

            for sortedVendorData in sortedVendorsData:

                lowestOfferPrice = float(sortedVendorData['sellingPrice'])

                try:

                    stock = sortedVendorData['buyableInventory']

                except:

                    stock = 0

                if stock > 0 and lowestOfferPrice > 0:

                    inStock = 1

                    break

            print lowestOfferPrice

            print inStock

            print "*************"

            if buyBoxPrice != lowestOfferPrice:

                isBuyBox = 0

            if inStock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestOfferPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print traceback.print_exc()

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id == 2:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']} 

            except:

                pass

            lowestSp = 0

            inStock = 0

            buyBoxPrice = 0

            isBuyBox = 0

            scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()

            try:

                result = scraperProductPage.read(data['identifier'])

                print result

                if result.get('lowestSp')!=0:

                    lowestSp = result.get('lowestSp')

                    inStock = result.get('inStock')

                    buyBoxPrice = result.get('buyBoxPrice')

            except:

                print "Unable to scrape product page ",data['identifier']

            print lowestSp

            print inStock

            if buyBoxPrice is not None and buyBoxPrice == lowestSp:

                isBuyBox = 1

            if lowestSp > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock,'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestSp = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now),'buyBoxFlag':isBuyBox}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id == 5:

        try:

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            except:

                pass

            url = data['marketPlaceUrl']

            lowestPrice = 0.0

            try:

                sc = ShopCluesScraper.ShopCluesScraper()

                url = transformUrl(url, 5)

                productInfo = sc.read(url)

            except Exception as e:

                traceback.print_exc()

                productInfo = {}

                productInfo['price'] = 0.0

                productInfo['inStock'] = 0

                productInfo['isCod'] = 0

                productInfo['coupon'] = ""

            print "LowestPrice ",productInfo['price']

            if productInfo['price'] > 0 and productInfo['inStock']==1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'],'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'],'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':productInfo['inStock']}})

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':productInfo['price'] , 'in_stock':productInfo['inStock'],'dealType':data['dealType'], 'rank':data['rank'],'codAvailable':productInfo['isCod']}})

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}})

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType'],'rank':data['rank'],'codAvailable':productInfo['isCod']}})

            return {'_id':data['_id'],'available_price':productInfo['price'],'in_stock':productInfo['inStock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'], 'coupon':productInfo['coupon'],'codAvailable':productInfo['isCod'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            traceback.print_exc()

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':5,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    elif source_id == 6:

        try:

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}

            except:

                pass

            paytmScraper = PaytmScraper.PaytmScraper()

            url = "https://catalog.paytm.com/v1/mobile/product/%s"%(data['identifier'].strip())

            try:

                result = paytmScraper.read(url)

                print result

                effective_price = result.get('offerPrice')

                gross_price = effective_price

                shareUrl = result.get('shareUrl')

                if shareUrl is None:

                    shareUrl = data['marketPlaceUrl']

                coupon = ""

                try:

                    offers = PaytmOfferScraper.fetchOffers(result['offerUrl'])

                    try:

                        addToPaytmMaster(offers.get('codes'))

                    except:

                        print "Error in adding coupon"

                        traceback.print_exc()

                    bestOffer = {}

                    for offer_data in offers.get('codes'):

                        if effective_price > offer_data.get('effective_price'):

                            effective_price = offer_data.get('effective_price')

                            bestOffer = offer_data

                            coupon = bestOffer.get('code')

                except:

                    pass

                """Temp fix"""

                if len(coupon) > 0:

                    result['codAvailable'] = 0

                available_price = effective_price 

                if result['inStock']:

                    inStock = 1

                    get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':available_price,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':1,'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon,'gross_price':gross_price,'marketPlaceUrl':shareUrl}})

                    get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':available_price , 'in_stock':1,'codAvailable':result.get('codAvailable'),'gross_price':gross_price}})

                else:

                    inStock = 0

                    get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon,'marketPlaceUrl':shareUrl}})

                    get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'codAvailable':result.get('codAvailable')}})

            except:

                inStock = 0

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1}})

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})

            return {'_id':data['_id'],'available_price':available_price,'in_stock':inStock,'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':shareUrl,'thumbnail':data['thumbnail'], 'coupon':coupon,'codAvailable':result['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':gross_price}

        except Exception as e:

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            traceback.print_exc()

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':6,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer'],'gross_price':data.get('gross_price')}

    elif source_id ==7:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data.get('ignorePricing') ==1:

                print "Ignored items returning for %d"%(data['_id'])

                return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

            try:

                if ignoreLastUpdated and data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'],'tagline': data['tagline'], 'offer': data['offer']}

            except Exception as e:

                print "Exception snapdeal"

                print e

            response = None

            try:

                scraper = HomeShop18Scraper.HomeShop18Scraper()

                response = scraper.read('http://m.homeshop18.com/product.mobi?productId=%s'%(str(data['identifier'])))

            except Exception as e:

                print e

                scraper = HomeShop18Scraper.HomeShop18Scraper()

                response = scraper.read('http://m.homeshop18.com/product.mobi?productId=%s'%(str(data['identifier'])))

            lowestOfferPrice = 0

            inStock = 0

            if response is not None:

                lowestOfferPrice = float(response['price']+response['shippingCharge'])

                inStock = response['inStock']          

            print lowestOfferPrice

            print inStock

            print "*************"

            if lowestOfferPrice ==0:

                inStock = 0

            if inStock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            else:

                lowestOfferPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

        except Exception as e:

            print traceback.print_exc()

            print "Exception for _id %d and source %s"%(data['_id'], source_id)

            print e

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':7,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}

    else:

        return {}

def recomputePoints(deal):

    try:

        nlcPoints = getNlcPoints(deal)

    except:

        traceback.print_exc()

        nlcPoints = deal['nlcPoints']

    bundleDealPoints = list(get_mongo_connection().Catalog.DealPoints.find({'skuBundleId':deal['skuBundleId'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))

    if len(bundleDealPoints) > 0:

        manualDealThresholdPrice = bundleDealPoints[0]['dealThresholdPrice']

        dealPoints = bundleDealPoints[0]['dealPoints']

    else:

        dealPoints = 0

        manualDealThresholdPrice = None

    if deal['available_price'] > manualDealThresholdPrice:

        dealPoints = 0

    if dealPoints != deal['dealPoints'] or manualDealThresholdPrice != deal['manualDealThresholdPrice'] or nlcPoints != deal['nlcPoints']:

        get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': manualDealThresholdPrice}})

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)

def recomputeDeal(item):

    """Lets recompute deal for this bundle"""

    print "Recomputing for bundleId",item.get('skuBundleId')

    skuBundleId = item['skuBundleId']

    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))

    bestPrice = float("inf")

    bestOne = None

    bestSellerPoints = 0

    toUpdate = []

    prepaidBestPrice = float("inf")

    prepaidBestOne = None

    prepaidBestSellerPoints = 0

    for similarItem in similarItems:

        try:

            recomputePoints(similarItem)

        except:

            traceback.print_exc()

        if similarItem['codAvailable'] ==1:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem['netPriceAfterCashBack'] < bestPrice:

                bestOne = similarItem

                bestPrice = similarItem['netPriceAfterCashBack']

                bestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem['netPriceAfterCashBack'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:

                bestOne = similarItem

                bestPrice = similarItem['netPriceAfterCashBack']

                bestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

        else:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem['netPriceAfterCashBack'] < prepaidBestPrice:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem['netPriceAfterCashBack']

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem['netPriceAfterCashBack'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem['netPriceAfterCashBack']

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

    #print "bestOne ", bestOne

    #print "prepaid best one",  prepaidBestOne

    if bestOne is not None or prepaidBestOne is not None:

        for similarItem in similarItems:

            toUpdate.append(similarItem['_id'])

        if bestOne is not None:

            toUpdate.remove(bestOne['_id'])

            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})

        if prepaidBestOne is not None:

            if bestOne is not None:

                if prepaidBestOne['netPriceAfterCashBack'] < bestOne['netPriceAfterCashBack']: 

                    toUpdate.remove(prepaidBestOne['_id'])

                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

            else:

                toUpdate.remove(prepaidBestOne['_id'])

                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

    if len(toUpdate) > 0:

        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

    print "Done with recomputing"

def getLatestPrice(skuBundleId, source_id):

    temp = []

    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))

    item = None

    for item in itemIds:

        item['dealFlag'] = 0

        item['dealType'] = 0

        item['dealUrl'] = ""

        if item['source_id'] ==5 and item['rank'] == 0 and item['category_id']!=6:

            continue

        if item['source_id'] ==3:

            item['marketPlaceUrl'] = item['marketPlaceUrl']+'?supc='+item.get('identifier')

        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))

        if len(manualDeals) > 0:

            item['dealFlag'] = 1

            item['dealType'] =manualDeals[0]['dealType']

            item['dealUrl'] = manualDeals[0]['dealUrl']

        info = returnLatestPrice(item, source_id)

        if item['source_id'] != SOURCE_MAP.get('PAYTM.COM'):

            data = calculateCashBack(item, info['available_price'])

        else:

            if info['codAvailable'] ==0:

                data = calculateCashBack(item, info['gross_price'])

            else:

                data = calculateCashBack(item, info['available_price'])  #No need, can be done in if with or clause.

        info['cash_back'] = data['cash_back']

        info['cash_back_type'] = data['cash_back_type']

        info['netPriceAfterCashBack'] = data['netPriceAfterCashBack']

        info['showNetPrice'] = data['showNetPrice']

        get_mongo_connection().Catalog.Deals.update({'_id':item['_id']},{"$set":{'netPriceAfterCashBack':info['netPriceAfterCashBack']}})

        temp.append(info)

    if item is not None:

        try:

            thread = threading.Thread(target=recomputeDeal, args = (item,))

            thread.daemon = True

            thread.start()    

        except:

            print traceback.print_exc()

            print "Unable to compute deal for ",skuBundleId

    return temp

def getLatestPriceById(id):

    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))

    item[0]['dealFlag'] = 0

    item[0]['dealType'] = 0

    item[0]['dealUrl'] = ""

    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item[0]['source_id'], 'sku':item[0]['_id']}))

    if len(manualDeals) > 0:

        item[0]['dealFlag'] = 1

        item[0]['dealType'] =manualDeals[0]['dealType']

        item[0]['dealUrl'] = manualDeals[0]['dealUrl']

    info = returnLatestPrice(item[0], item[0]['source_id'])

    if item[0]['source_id'] != SOURCE_MAP.get('PAYTM.COM'):

        data = calculateCashBack(item[0], info['available_price'])

    else:

        if info['codAvailable'] ==0:

            data = calculateCashBack(item[0], info['gross_price'])

        else:

            data = calculateCashBack(item[0], info['available_price'])  #No need, can be done in if with or clause.

    info['cash_back'] = data['cash_back']

    info['cash_back_type'] = data['cash_back_type']

    info['netPriceAfterCashBack'] = data['netPriceAfterCashBack']

    info['showNetPrice'] = data['showNetPrice']

    get_mongo_connection().Catalog.Deals.update({'_id':item[0]['_id']},{"$set":{'netPriceAfterCashBack':info['netPriceAfterCashBack']}})

    print info

    try:

        thread = threading.Thread(target=recomputeDeal, args = (item[0],))

        thread.daemon = True

        thread.start()    

    except:

        print "Unable to compute deal for ",item[0]['skuBundleId']

    return info

def updatePriceForNotificationBundles(skuBundleId):

    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'priceUpdatedOn':{'$lte':to_java_date(datetime.now() - timedelta(minutes=2))},'source_id':{"$in":SOURCE_MAP.values()}}))

    for item in itemIds:

        print item['_id']

        item['dealFlag'] = 0

        item['dealType'] = 0

        item['dealUrl'] = ""

        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item['source_id'], 'sku':item['_id']}))

        if len(manualDeals) > 0:

            item['dealFlag'] = 1

            item['dealType'] =manualDeals[0]['dealType']

            item['dealUrl'] = manualDeals[0]['dealUrl']

        info = returnLatestPrice(item, item['source_id'],False)

        print info

def main():

    print datetime.now()

    print "retuned %s"%(str(getLatestPrice(5899,4)))

    print datetime.now()

if __name__=='__main__':

    main()