Subversion Repositories SmartDukaan

import pymongo

from elixir import *

from shop2020.model.v1.catalog.impl import DataService

from shop2020.model.v1.catalog.impl.DataService import Item

from shop2020.clients.InventoryClient import InventoryClient

from dtr.utils.utils import to_java_date, getCashBack

from datetime import datetime, timedelta

import time

import optparse

from dtr.storage.MemCache import MemCache

import traceback

parser = optparse.OptionParser()

parser.add_option("-H", "--host", dest="hostname",

                      default="localhost",

                      type="string", help="The HOST where the DB server is running",

                      metavar="host")

parser.add_option("-m", "--m", dest="mongoHost",

                      default="localhost",

                      type="string", help="The HOST where the mongo server is running",

                      metavar="mongo_host")

(options, args) = parser.parse_args()

mc = MemCache(options.mongoHost)

DataService.initialize(db_hostname=options.hostname)

con = None

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6,'HOMESHOP18.COM':7}

DISCOUNT_TYPE = {'MRP':1,'DP':2}

LATEST_UPDATED_ITEMS = []

STATUS_WEIGHTAGE = {1 : 1.0, 2 : 2.0, 3 : 1.0, 4 : 0.5}

DEAL_POINTS_MAP = {}

now = datetime.now()

class __DealPoints:

    def __init__(self, manualDealThresholdPrice, points):

        self.manualDealThresholdPrice = manualDealThresholdPrice

        self.points = points

class __SkuInfo:

    def __init__(self, _id, skuBundleId, category_id, mrp, available_price, source_id, rank, maxNlc, minNlc, schemeAmount, minDiscount, \

                 maxDiscount, discountType, dp, nlcPoints, status, in_stock, maxprice, brand, dealType, brand_id, manualDealThresholdPrice,\

                 codAvailable,showDp,gross_price, subCategoryId,subCategory,shippingCost,netPriceAfterCashBack):

        self._id = _id

        self.skuBundleId = skuBundleId

        self.category_id = category_id

        self.mrp = mrp

        self.available_price = available_price

        self.source_id = source_id

        self.rank = rank

        self.maxNlc = maxNlc

        self.minNlc = minNlc

        self.schemeAmount = schemeAmount

        self.minDiscount = minDiscount

        self.maxDiscount = maxDiscount

        self.discountType = discountType

        self.dp = dp

        self.nlcPoints = nlcPoints

        self.status = status

        self.in_stock = in_stock

        self.maxprice = maxprice

        self.brand = brand

        self.dealType = dealType

        self.brand_id = brand_id

        self.manualDealThresholdPrice = manualDealThresholdPrice

        self.codAvailable = codAvailable

        self.showDp = showDp

        self.gross_price = gross_price

        self.subCategoryId = subCategoryId

        self.subCategory = subCategory

        self.shippingCost = shippingCost

        self.netPriceAfterCashBack = netPriceAfterCashBack

def get_mongo_connection(host=options.mongoHost, port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def populateStuff():

    print "Inside populate"

    global LATEST_UPDATED_ITEMS

    """Fetch latest updated items across portals

       and calculate max and min R-Nlc"""

    offset= 0

    while(True):

        print "Fetching records offset %d and limit %d" %(offset,300)

        topSkus = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'updatedOn': { "$gt": to_java_date(now - timedelta(hours=1))} }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).skip(offset).limit(300))

        if len((topSkus)) == 0:

            break

        #topSkus = collection.find( {'_id':664})

        for sku in topSkus:

            """Fix this """

            #TODO Compute deal flags else where.

            if sku['source_id'] != SOURCE_MAP.get('PAYTM.COM'):

                netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['available_price'])

            else:

                if sku['codAvailable'] ==0:

                    netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['gross_price'])

                else:

                    netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['available_price'])

            info = __SkuInfo(sku['_id'], sku['skuBundleId'], sku['category_id'], sku['mrp'], sku['available_price'], sku['source_id'], sku['rank'], None, None, 0.0, None, \

                             None, None, None, None, sku['status'], sku['in_stock'],sku['maxPrice'],sku['brand'].strip().upper(), 0, sku['brand_id'], None, sku['codAvailable'], 0, sku['gross_price'], sku['subCategoryId'], \

                             sku['subCategory'],sku['shippingCost'],netPriceAfterCashBack)

            exceptionalNlc = list(get_mongo_connection().Catalog.ExceptionalNlc.find( {"$and" : [ {'skuBundleId':info.skuBundleId}, {'overrideNlc':1} ]} ))

            if len(exceptionalNlc) > 0:

                """Exceptional nlc found, no need to calculate max and min R-nlc"""

                info.maxNlc = exceptionalNlc[0]['maxNlc']

                info.minNlc = exceptionalNlc[0]['minNlc']

                if info.maxprice == 0:

                    info.maxprice = exceptionalNlc[0]['maxNlc']

                LATEST_UPDATED_ITEMS.append(info)

                continue

            skuSchemeDetails = list(get_mongo_connection().Catalog.SkuSchemeDetails.find( {'skuBundleId':info.skuBundleId}))

            if len(skuSchemeDetails) > 0:

                """Sku scheme details, populate scheme amount (Recently added)"""

                #TODO Add start date and end date of scehems

                info.schemeAmount = float(skuSchemeDetails[0]['schemeAmount'])

            skuDealerPrices = list(get_mongo_connection().Catalog.SkuDealerPrices.find( {'skuBundleId':info.skuBundleId} ) )

            if len(skuDealerPrices) > 0:

                info.dp = skuDealerPrices[0]['dp']

                info.showDp = skuDealerPrices[0]['showDp']

            skuDiscount = list(get_mongo_connection().Catalog.SkuDiscountInfo.find( {'skuBundleId':info.skuBundleId} ) )

            if len(skuDiscount) > 0:

                """Sku rule found, populate max , min Discount and discount type"""

                info.maxDiscount = skuDiscount[0]['max_discount']

                info.minDiscount = skuDiscount[0]['min_discount']

                info.discountType = DISCOUNT_TYPE.get(skuDiscount[0]['discountType'].upper())

                LATEST_UPDATED_ITEMS.append(info)

                continue

            categoryDiscount = list(get_mongo_connection().Catalog.CategoryDiscount.find( {"$and" : [{'brand':sku['brand'].strip().upper()}, {'category_id':sku['category_id']} ]} ))

            if len(categoryDiscount) > 0:

                info.maxDiscount = categoryDiscount[0]['max_discount']

                info.minDiscount = categoryDiscount[0]['min_discount']

                info.discountType = DISCOUNT_TYPE.get(categoryDiscount[0]['discountType'].upper())

            LATEST_UPDATED_ITEMS.append(info)

        offset = offset + 300

    for lol in LATEST_UPDATED_ITEMS:

        print lol.__dict__

def calculateNlc():

    global LATEST_UPDATED_ITEMS

    populated = 0

    while(populated <= len(LATEST_UPDATED_ITEMS)):

        inventory_client = InventoryClient().get_client()

        for obj in LATEST_UPDATED_ITEMS[populated:300+populated]:

            if obj.maxNlc > 0 and obj.minNlc > 0:

                continue

            saholic_sku = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'skuBundleId': obj.skuBundleId}, { 'source_id' : SOURCE_MAP.get('SAHOLIC')}] }))

            identifier = None

            if len(saholic_sku) > 0:

                identifier = saholic_sku[0]['identifier']

            if obj.discountType == DISCOUNT_TYPE.get('MRP'):

                if obj.mrp == 0:

                    """Now mrp is zero, so we have to use saholic MRP"""

                    if identifier is not None:

                        it = Item.query.filter_by(catalog_item_id=identifier).first()

                        obj.mrp = it.mrp

                if obj.mrp > 0:

                    print obj._id

                    obj.minNlc = obj.mrp - (obj.mrp * obj.maxDiscount/100) - obj.schemeAmount

                    obj.maxNlc = obj.mrp - (obj.mrp * obj.minDiscount/100) - obj.schemeAmount

                    if obj.maxprice == 0:

                        obj.maxprice = obj.maxNlc  

            elif obj.discountType == DISCOUNT_TYPE.get('DP'):

                if obj.dp == 0:

                    """Now dp is zero, so we have to use saholic minimum dp for item"""

                    if identifier is not None:

                        it = Item.query.filter_by(catalog_item_id=identifier).first()

                        try:

                            vendorPricing = inventory_client.getAllItemPricing(it.id)

                            min_dp = min(pricing.dealerPrice for pricing in vendorPricing)

                            obj.dp = min_dp 

                            obj.showDp = 1

                        except:

                            pass

                if obj.dp > 0:

                    obj.minNlc = obj.dp - (obj.dp * obj.maxDiscount/100) - obj.schemeAmount

                    obj.maxNlc = obj.dp - (obj.dp * obj.minDiscount/100) - obj.schemeAmount

                    if obj.maxprice == 0:

                        obj.maxprice = obj.maxNlc  

            else:

                """No rule found, use saholic min nlc as max and min R-Nlc"""

                if identifier is not None:

                    it = Item.query.filter_by(catalog_item_id=identifier).first()

                    try:

                        vendorPricing = inventory_client.getAllItemPricing(it.id)

                        min_nlc = min(pricing.nlc for pricing in vendorPricing)

                        obj.maxNlc = min_nlc

                        obj.minNlc = min_nlc

                        if obj.maxprice == 0:

                            obj.maxprice = obj.maxNlc   

                    except:

                        pass

        populated = populated + 300

        time.sleep(10) 

def addManualDealsInfo():

    for sku in LATEST_UPDATED_ITEMS:

        manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':sku.source_id, 'sku':sku._id}))

        if len(manualDeal) > 0:

            sku.dealType = manualDeal[0]['dealType']

    """Remove deal flag from expired deals"""

    manualDeals = list(get_mongo_connection().Catalog.Deals.find({'dealType':1}))

    for manualDeal in manualDeals:

        d_manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':manualDeal['source_id'], 'sku':manualDeal['_id']}))

        if len(d_manualDeal) > 0:

            continue

        else:

            get_mongo_connection().Catalog.Deals.update({'_id':manualDeal['_id']},{"$set":{'dealType':0}},upsert=False, multi=False)

def calculateNlcPoints():

    global LATEST_UPDATED_ITEMS

    print "inside nlc oints"

    for sku in LATEST_UPDATED_ITEMS:

        if sku.maxNlc and sku.minNlc:

            print sku._id

            """Create map - TODO"""

            if sku.status == 2:

                eolWeight = .60

            else:

                eolWeight = 1.0

            if sku.category_id == 3:

                basePointPercentage = 5.0

                maxNlcPoints = 200

            elif sku.category_id == 5:

                basePointPercentage = 8.0

                maxNlcPoints = 150

            elif sku.category_id == 6:

                basePointPercentage = 5.0

                maxNlcPoints = 200

            else:

                basePointPercentage = 10.0

                maxNlcPoints = 150

            discFromMinNlc = float((sku.minNlc - sku.available_price))/sku.available_price *100

            discFromMaxNlc = float((sku.maxNlc - sku.available_price))/sku.available_price *100

            print discFromMinNlc

            print discFromMaxNlc

            if discFromMinNlc > 0:

                nlcPoints = 100/basePointPercentage * discFromMinNlc

            elif discFromMinNlc < 0 and discFromMaxNlc > 0:

                nlcPoints = 0

            else:

                nlcPoints = 100/basePointPercentage * discFromMinNlc

            if (min(nlcPoints,maxNlcPoints)) > 0:

                sku.nlcPoints = (min(nlcPoints,maxNlcPoints)) * eolWeight

            else:

                sku.nlcPoints = (min(nlcPoints,maxNlcPoints))

        else:

            sku.nlcPoints = 0

def commitData():

    global LATEST_UPDATED_ITEMS

    for sku in LATEST_UPDATED_ITEMS:

        #get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{'$set' : sku.__dict__},upsert=True,multi=True)

        get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{"$set":sku.__dict__},upsert=True)

def addBestSellerPoints():

    allItems = list(get_mongo_connection().Catalog.Deals.find({})) 

    for sku in allItems:

        bestSellerPoints = list(get_mongo_connection().Catalog.BestSellerPoints.find( {"$and":[{'min_rank': { "$lte": sku['rank'] } }, {'max_rank': { "$gte": sku['rank'] } } , { 'category_id' : sku['category_id'] }, { 'source_id' : sku['source_id'] }] } ))

        if len(bestSellerPoints) > 0:

            print bestSellerPoints[0]['points']

            if (bestSellerPoints[0]['points']) > 0:

                sku['bestSellerPoints'] = (bestSellerPoints[0]['points']) * bestSellerPoints[0]['weightage'] * STATUS_WEIGHTAGE.get(sku['status'])

            else:

                sku['bestSellerPoints'] = (bestSellerPoints[0]['points'])

        else:

            if sku['category_id'] == 6:

                sku['bestSellerPoints'] = 0

            else:

                sku['bestSellerPoints'] = -120

        #sku['totalPoints'] = sku['bestSellerPoints'] + sku['nlcPoints']

        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'bestSellerPoints':sku['bestSellerPoints']}},multi=False)

    shortageSkus = get_mongo_connection().Catalog.MasterData.find({"$and":[{'is_shortage': 1 }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).distinct('_id')

    print "Shortage skus"

    print shortageSkus

    for sku in allItems:

        deal_item = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':sku['skuBundleId']}).sort('bestSellerPoints',pymongo.DESCENDING).limit(1))

        sku['catalogBestSellerPoints'] = deal_item[0]['bestSellerPoints']

        shortagePoints = 50 if sku['_id'] in shortageSkus else 0

        print "Shortage points for ",sku['_id']

        print shortagePoints

        dealPoints = DEAL_POINTS_MAP.get(sku['skuBundleId'])

        if dealPoints is not None and dealPoints.manualDealThresholdPrice >= sku['available_price']:

            sku['dealPoints'] = dealPoints.points

            sku['manualDealThresholdPrice'] = dealPoints.manualDealThresholdPrice

        else:

            sku['dealPoints'] = 0

            sku['manualDealThresholdPrice'] = None

        sku['totalPoints'] = sku['catalogBestSellerPoints'] + sku['nlcPoints'] + shortagePoints + sku['dealPoints'] 

        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'catalogBestSellerPoints':sku['catalogBestSellerPoints'],'totalPoints':sku['totalPoints'],'dealPoints':sku['dealPoints'], \

                                                                                    'manualDealThresholdPrice':sku['manualDealThresholdPrice']}},multi=False)

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)   

def eliminateSimilarDeals():

    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')

    for skuBundleId in allItems:

        print skuBundleId

        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))

        bestPrice = float("inf")

        bestOne = None

        bestSellerPoints = 0

        toUpdate = []

        prepaidBestPrice = float("inf")

        prepaidBestOne = None

        prepaidBestSellerPoints = 0

        for similarItem in similarItems:

            if similarItem['codAvailable'] ==1:

                if mc.get("negative_deals") is None:

                    populateNegativeDeals()

                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                    continue

                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:

                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                    continue

                if similarItem['netPriceAfterCashBack'] < bestPrice:

                    bestOne = similarItem

                    bestPrice = similarItem['netPriceAfterCashBack']

                    bestSellerPoints = similarItem['bestSellerPoints']

                elif similarItem['netPriceAfterCashBack'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:

                    bestOne = similarItem

                    bestPrice = similarItem['netPriceAfterCashBack']

                    bestSellerPoints = similarItem['bestSellerPoints']

                else:

                    pass

            else:

                if mc.get("negative_deals") is None:

                    populateNegativeDeals()

                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                    continue

                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:

                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                    continue

                if similarItem['netPriceAfterCashBack'] < prepaidBestPrice:

                    prepaidBestOne = similarItem

                    prepaidBestPrice = similarItem['netPriceAfterCashBack']

                    prepaidBestSellerPoints = similarItem['bestSellerPoints']

                elif similarItem['netPriceAfterCashBack'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:

                    prepaidBestOne = similarItem

                    prepaidBestPrice = similarItem['netPriceAfterCashBack']

                    prepaidBestSellerPoints = similarItem['bestSellerPoints']

                else:

                    pass

        if bestOne is not None or prepaidBestOne is not None:

            for similarItem in similarItems:

                toUpdate.append(similarItem['_id'])

            if bestOne is not None:

                toUpdate.remove(bestOne['_id'])

                get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})

            if prepaidBestOne is not None:

                if bestOne is not None:

                    if prepaidBestOne['netPriceAfterCashBack'] < bestOne['netPriceAfterCashBack']: 

                        toUpdate.remove(prepaidBestOne['_id'])

                        get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

                else:

                    toUpdate.remove(prepaidBestOne['_id'])

                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

        if len(toUpdate) > 0:

            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

def populateDealPointsBundle():

    global DEAL_POINTS_MAP

    activeDealPoints = get_mongo_connection().Catalog.DealPoints.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})

    for activeDeal in activeDealPoints:

        dealPoints = __DealPoints(activeDeal['dealThresholdPrice'], activeDeal['dealPoints'])

        DEAL_POINTS_MAP[activeDeal['skuBundleId']] = dealPoints

def getNetPriceForItem(itemId, source_id, category_id ,price):

    cash_back_type = 0

    cash_back = 0

    try:

        cashBack = getCashBack(itemId, source_id, category_id, mc, options.mongoHost)

        if not cashBack or cashBack.get('cash_back_status')!=1:

            cash_back_type = 0

            cash_back = 0 

        else:

            if cashBack['cash_back_type'] in (1,2):

                if cashBack.get('maxCashBack') is not None:

                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*price)/100 > cashBack.get('maxCashBack'):

                        cashBack['cash_back_type'] = 2

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    else:

                        pass

                cash_back_type = cashBack['cash_back_type']

                cash_back = float(cashBack['cash_back'])

    except Exception as cashBackEx:

        pass

    if cash_back_type ==1:

        return (price - float(cash_back)*price/100)

    elif cash_back_type ==2:

        return (price - cash_back)

    else:

        return price

# def calculateNetPriceAfterCashback():

#     for data in LATEST_UPDATED_ITEMS:

#         master = list(get_mongo_connection().Catalog.MasterData.find({'_id':data._id}))

#         if master[0]['source_id'] != SOURCE_MAP.get('PAYTM.COM'):

#             netPriceAfterCashBack = getNetPriceForItem(data._id, master[0]['source_id'], master[0]['category_id'], master[0]['available_price'])

#             get_mongo_connection().Catalog.Deals.update({'_id':data._id},{"$set":{'available_price':master[0]['available_price'],'in_stock':master[0]['in_stock'],'netPriceAfterCashBack':netPriceAfterCashBack}})

#         else:

#             if master[0]['codAvailable'] ==0:

#                 netPriceAfterCashBack = getNetPriceForItem(data._id, master[0]['source_id'], master[0]['category_id'], master[0]['gross_price'])

#             else:

#                 netPriceAfterCashBack = getNetPriceForItem(data._id, master[0]['source_id'], master[0]['category_id'], master[0]['available_price'])

#             get_mongo_connection().Catalog.Deals.update({'_id':data._id},{"$set":{'available_price':master[0]['available_price'],'gross_price':master[0]['gross_price'],'in_stock':master[0]['in_stock'],'netPriceAfterCashBack':netPriceAfterCashBack}})

#         

def main():

    try:

        populateStuff()

        calculateNlc()

        addManualDealsInfo()

    finally:

        session.close()

    calculateNlcPoints()

    commitData()

    populateDealPointsBundle()

    addBestSellerPoints()

    eliminateSimilarDeals()

if __name__=='__main__':

    main()