Subversion Repositories SmartDukaan

import pymongo

from dtr.utils.utils import to_java_date, DEAL_PRIORITY

from datetime import datetime, timedelta

import optparse

from dtr.storage.MemCache import MemCache

from dtr.utils.utils import getCashBack

from shop2020.model.v1.catalog.script import AmazonAsyncScraper

import traceback

con = None

parser = optparse.OptionParser()

parser.add_option("-m", "--m", dest="mongoHost",

                      default="localhost",

                      type="string", help="The HOST where the mongo server is running",

                      metavar="mongo_host")

(options, args) = parser.parse_args()

mc = MemCache(options.mongoHost)

amScraper = AmazonAsyncScraper.Products("AKIAII3SGRXBJDPCHSGQ", "B92xTbNBTYygbGs98w01nFQUhbec1pNCkCsKVfpg", "AF6E3O0VE0X4D")

marketplaceId = 'A21TJRUUN4KGV'

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6}

def get_mongo_connection(host=options.mongoHost, port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def getNetPriceForItem(itemId, source_id, category_id ,price):

    cash_back_type = 0

    cash_back = 0

    try:

        cashBack = getCashBack(itemId, source_id, category_id, mc, options.mongoHost)

        if not cashBack or cashBack.get('cash_back_status')!=1:

            cash_back_type = 0

            cash_back = 0 

        else:

            if cashBack['cash_back_type'] in (1,2):

                if cashBack.get('maxCashBack') is not None:

                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*price)/100 > cashBack.get('maxCashBack'):

                        cashBack['cash_back_type'] = 2

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    else:

                        pass

                cash_back_type = cashBack['cash_back_type']

                cash_back = float(cashBack['cash_back'])

    except Exception as cashBackEx:

        pass

    if cash_back_type ==1:

        return (price - float(cash_back)*price/100)

    elif cash_back_type ==2:

        return (price - cash_back)

    else:

        return price

def populate():

    toScrapMap = {}

    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))

    for bestSeller in bestSellers: 

        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))

        for data in amazonBestSellers:

            if not toScrapMap.has_key(data['_id']):

                data['dealFlag'] = 0

                data['dealType'] = 0

                data['dealUrl'] = ""

                toScrapMap[data['_id']] = data

    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':-100}}))

    for deal in dealFlagged:

        if not toScrapMap.has_key(deal['_id']):

            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))

            data[0]['dealFlag'] = 0

            data[0]['dealType'] = 0

            data[0]['dealUrl'] = ""

            toScrapMap[deal['_id']] = data[0]

    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':1}))

    for manualDeal in manualDeals:

        if not toScrapMap.has_key(manualDeal['sku']):

            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))

            if len(data) > 0:

                data[0]['dealFlag'] = 1

                data[0]['dealType'] = manualDeal['dealType']

                data[0]['dealUrl'] = manualDeal['dealUrl']

                toScrapMap[manualDeal['sku']] = data[0]

        else:

            data = toScrapMap.get(manualDeal['sku'])

            data['dealFlag'] = 1

            data['dealType'] = manualDeal['dealType']

            data['dealUrl'] = manualDeal['dealUrl']

    count = 0

    temp = {}

    for v in toScrapMap.itervalues():

        print v

        #Lets validate identifier

        if len(str(v['identifier']).strip()) !=10:

            continue

        temp[str(v['identifier']).strip().upper()] = v

        count = count+1

        if count == 20:

            scrapeAmazon(temp)

            temp = {}

            count =0

def scrapeAmazon(dataMap):

    asinPricingMap = amScraper.get_competitive_pricing_for_asin(marketplaceId, dataMap.keys())

    print "asinPricingMap ",asinPricingMap

    for k, data in dataMap.iteritems():

        print data

        inStock = 0

        print str(data['identifier'])

        if data['identifier'] is None or len(data['identifier'].strip())==0:

            return

        if data.get('ignorePricing') ==1:

            print "Ignored items returning for %d"%(data['_id'])

            return 

        try:

            if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):

                print "sku id is already updated",data['_id'] 

                return

        except:

            pass

        lowestPrice = asinPricingMap.get(k)    

        print lowestPrice

        if lowestPrice > 0:

            inStock = 1

            netPriceAfterCashBack = getNetPriceForItem(data['_id'], SOURCE_MAP.get('AMAZON'), data['category_id'], lowestPrice)

        else:

            netPriceAfterCashBack = getNetPriceForItem(data['_id'], SOURCE_MAP.get('AMAZON'), data['category_id'], data['available_price'])

        print lowestPrice

        print inStock

        if lowestPrice > 0:

            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)

            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType'],'codAvailable':data['codAvailable'],'netPriceAfterCashBack':netPriceAfterCashBack}}, multi=True)

        else:

            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)

            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'dealType':data['dealType'],'codAvailable':data['codAvailable'], 'netPriceAfterCashBack':netPriceAfterCashBack}})

        try:

            recomputeDeal(data)

        except:

            print "Unable to compute deal for ",data['skuBundleId']    

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)

#def recomputePoints(item, deal):

#    try:

#        if item.get('available_price') == deal['available_price']:

#            print "No need to compute points for %d , as price is still same" %(item['_id'])

#            raise

#        nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])

#    except:

#        traceback.print_exc()

#        nlcPoints = deal['nlcPoints']

#    

#    bundleDealPoints = list(get_mongo_connection().Catalog.DealPoints.find({'skuBundleId':item['skuBundleId'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))

#    if len(bundleDealPoints) > 0:

#        item['manualDealThresholdPrice'] = bundleDealPoints[0]['dealThresholdPrice']

#        dealPoints = bundleDealPoints[0]['dealPoints']

#    else:

#        dealPoints = 0

#        item['manualDealThresholdPrice'] = None

#    

#    get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})

def recomputeDeal(item):

    """Lets recompute deal for this bundle"""

    print "Recomputing for bundleId %d" %(item.get('skuBundleId'))

    skuBundleId = item['skuBundleId']

    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))

    bestPrice = float("inf")

    bestOne = None

    bestSellerPoints = 0

    toUpdate = []

    prepaidBestPrice = float("inf")

    prepaidBestOne = None

    prepaidBestSellerPoints = 0

    for similarItem in similarItems:

        if similarItem['codAvailable'] ==1:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem.get('netPriceAfterCashBack') < bestPrice:

                bestOne = similarItem

                bestPrice = similarItem.get('netPriceAfterCashBack')

                bestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem.get('netPriceAfterCashBack') == bestPrice:

                try:

                    if (DEAL_PRIORITY.index(int(similarItem['source_id'])) > DEAL_PRIORITY.index(int(bestOne['source_id']))):

                        continue

                except:

                    traceback.print_exc()

                bestOne = similarItem

                bestPrice = similarItem.get('netPriceAfterCashBack')

                bestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

        else:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem.get('netPriceAfterCashBack') < prepaidBestPrice:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem.get('netPriceAfterCashBack')

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem.get('netPriceAfterCashBack') == prepaidBestPrice:

                try:

                    if (DEAL_PRIORITY.index(int(similarItem['source_id'])) > DEAL_PRIORITY.index(int(bestOne['source_id']))):

                        continue

                except:

                    traceback.print_exc()

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem.get('netPriceAfterCashBack')

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

    if bestOne is not None or prepaidBestOne is not None:

        for similarItem in similarItems:

            toUpdate.append(similarItem['_id'])

        if bestOne is not None:

            toUpdate.remove(bestOne['_id'])

            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})

        if prepaidBestOne is not None:

            if bestOne is not None:

                if prepaidBestOne.get('netPriceAfterCashBack') < bestOne.get('netPriceAfterCashBack'): 

                    toUpdate.remove(prepaidBestOne['_id'])

                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

            else:

                toUpdate.remove(prepaidBestOne['_id'])

                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

    if len(toUpdate) > 0:

        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

def main():

    populate()

if __name__=='__main__':

    main()