Subversion Repositories SmartDukaan

import pymongo
from elixir import *
from shop2020.model.v1.catalog.impl import DataService
from shop2020.model.v1.catalog.impl.DataService import Item
from shop2020.clients.InventoryClient import InventoryClient
from dtr.utils.utils import to_java_date
from datetime import datetime, timedelta
import time
import optparse
from dtr.storage.MemCache import MemCache
import traceback

parser = optparse.OptionParser()
parser.add_option("-H", "--host", dest="hostname",
                      default="localhost",
                      type="string", help="The HOST where the DB server is running",
                      metavar="host")
parser.add_option("-m", "--m", dest="mongoHost",
                      default="localhost",
                      type="string", help="The HOST where the mongo server is running",
                      metavar="mongo_host")

(options, args) = parser.parse_args()

mc = MemCache(options.mongoHost)

DataService.initialize(db_hostname=options.hostname)

con = None
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6,'HOMESHOP18.COM':7}
DISCOUNT_TYPE = {'MRP':1,'DP':2}
LATEST_UPDATED_ITEMS = []
STATUS_WEIGHTAGE = {1 : 1.0, 2 : 2.0, 3 : 1.0, 4 : 0.5}

DEAL_POINTS_MAP = {}

now = datetime.now()

class __DealPoints:
    
    def __init__(self, manualDealThresholdPrice, points):
        self.manualDealThresholdPrice = manualDealThresholdPrice
        self.points = points
        

class __SkuInfo:
    
    def __init__(self, _id, skuBundleId, category_id, mrp, available_price, source_id, rank, maxNlc, minNlc, schemeAmount, minDiscount, \
                 maxDiscount, discountType, dp, nlcPoints, status, in_stock, maxprice, brand, dealType, brand_id, manualDealThresholdPrice,\
                 codAvailable,showDp,gross_price, subCategoryId,subCategory,shippingCost):
        self._id = _id
        self.skuBundleId = skuBundleId
        self.category_id = category_id
        self.mrp = mrp
        self.available_price = available_price
        self.source_id = source_id
        self.rank = rank
        self.maxNlc = maxNlc
        self.minNlc = minNlc
        self.schemeAmount = schemeAmount
        self.minDiscount = minDiscount
        self.maxDiscount = maxDiscount
        self.discountType = discountType
        self.dp = dp
        self.nlcPoints = nlcPoints
        self.status = status
        self.in_stock = in_stock
        self.maxprice = maxprice
        self.brand = brand
        self.dealType = dealType
        self.brand_id = brand_id
        self.manualDealThresholdPrice = manualDealThresholdPrice
        self.codAvailable = codAvailable
        self.showDp = showDp
        self.gross_price = gross_price
        self.subCategoryId = subCategoryId
        self.subCategory = subCategory
        self.shippingCost = shippingCost


def get_mongo_connection(host=options.mongoHost, port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def populateStuff():
    print "Inside populate"
    global LATEST_UPDATED_ITEMS
    """Fetch latest updated items across portals
       and calculate max and min R-Nlc"""
    offset= 0
    while(True):
        print "Fetching records offset %d and limit %d" %(offset,300)
        topSkus = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'updatedOn': { "$gt": to_java_date(now - timedelta(hours=1))} }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).skip(offset).limit(300))
        if len((topSkus)) == 0:
            break
        #topSkus = collection.find( {'_id':664})
        for sku in topSkus:
            """Fix this """
            #TODO Compute deal flags else where.
            info = __SkuInfo(sku['_id'], sku['skuBundleId'], sku['category_id'], sku['mrp'], sku['available_price'], sku['source_id'], sku['rank'], None, None, 0.0, None, \
                             None, None, None, None, sku['status'], sku['in_stock'],sku['maxPrice'],sku['brand'].strip().upper(), 0, sku['brand_id'], None, sku['codAvailable'], 0, sku['gross_price'], sku['subCategoryId'], \
                             sku['subCategory'],sku['shippingCost'])
            exceptionalNlc = list(get_mongo_connection().Catalog.ExceptionalNlc.find( {"$and" : [ {'skuBundleId':info.skuBundleId}, {'overrideNlc':1} ]} ))
            if len(exceptionalNlc) > 0:
                """Exceptional nlc found, no need to calculate max and min R-nlc"""
                info.maxNlc = exceptionalNlc[0]['maxNlc']
                info.minNlc = exceptionalNlc[0]['minNlc']
                if info.maxprice == 0:
                    info.maxprice = exceptionalNlc[0]['maxNlc']
                LATEST_UPDATED_ITEMS.append(info)
                continue
            
            skuSchemeDetails = list(get_mongo_connection().Catalog.SkuSchemeDetails.find( {'skuBundleId':info.skuBundleId}))
            if len(skuSchemeDetails) > 0:
                """Sku scheme details, populate scheme amount (Recently added)"""
                
                #TODO Add start date and end date of scehems
                
                info.schemeAmount = float(skuSchemeDetails[0]['schemeAmount'])
                
            skuDealerPrices = list(get_mongo_connection().Catalog.SkuDealerPrices.find( {'skuBundleId':info.skuBundleId} ) )
            if len(skuDealerPrices) > 0:
                info.dp = skuDealerPrices[0]['dp']
                info.showDp = skuDealerPrices[0]['showDp']
            skuDiscount = list(get_mongo_connection().Catalog.SkuDiscountInfo.find( {'skuBundleId':info.skuBundleId} ) )
            if len(skuDiscount) > 0:
                """Sku rule found, populate max , min Discount and discount type"""
                info.maxDiscount = skuDiscount[0]['max_discount']
                info.minDiscount = skuDiscount[0]['min_discount']
                info.discountType = DISCOUNT_TYPE.get(skuDiscount[0]['discountType'].upper())
                LATEST_UPDATED_ITEMS.append(info)
                continue
            
            categoryDiscount = list(get_mongo_connection().Catalog.CategoryDiscount.find( {"$and" : [{'brand':sku['brand'].strip().upper()}, {'category_id':sku['category_id']} ]} ))
            if len(categoryDiscount) > 0:
                info.maxDiscount = categoryDiscount[0]['max_discount']
                info.minDiscount = categoryDiscount[0]['min_discount']
                info.discountType = DISCOUNT_TYPE.get(categoryDiscount[0]['discountType'].upper())
        
            LATEST_UPDATED_ITEMS.append(info)
        offset = offset + 300
    for lol in LATEST_UPDATED_ITEMS:
        print lol.__dict__
    
    
def calculateNlc():
    global LATEST_UPDATED_ITEMS
    populated = 0
    while(populated <= len(LATEST_UPDATED_ITEMS)):
        inventory_client = InventoryClient().get_client()
        for obj in LATEST_UPDATED_ITEMS[populated:300+populated]:
            if obj.maxNlc > 0 and obj.minNlc > 0:
                continue
            saholic_sku = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'skuBundleId': obj.skuBundleId}, { 'source_id' : SOURCE_MAP.get('SAHOLIC')}] }))
            identifier = None
            if len(saholic_sku) > 0:
                identifier = saholic_sku[0]['identifier']
            if obj.discountType == DISCOUNT_TYPE.get('MRP'):
                if obj.mrp == 0:
                    """Now mrp is zero, so we have to use saholic MRP"""
                    if identifier is not None:
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
                        obj.mrp = it.mrp
                if obj.mrp > 0:
                    print obj._id
                    obj.minNlc = obj.mrp - (obj.mrp * obj.maxDiscount/100) - obj.schemeAmount
                    obj.maxNlc = obj.mrp - (obj.mrp * obj.minDiscount/100) - obj.schemeAmount
                    if obj.maxprice == 0:
                        obj.maxprice = obj.maxNlc  
            elif obj.discountType == DISCOUNT_TYPE.get('DP'):
                if obj.dp == 0:
                    """Now dp is zero, so we have to use saholic minimum dp for item"""
                    if identifier is not None:
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
                        try:
                            vendorPricing = inventory_client.getAllItemPricing(it.id)
                            min_dp = min(pricing.dealerPrice for pricing in vendorPricing)
                            obj.dp = min_dp 
                            obj.showDp = 1
                        except:
                            pass
                if obj.dp > 0:
                    obj.minNlc = obj.dp - (obj.dp * obj.maxDiscount/100) - obj.schemeAmount
                    obj.maxNlc = obj.dp - (obj.dp * obj.minDiscount/100) - obj.schemeAmount
                    if obj.maxprice == 0:
                        obj.maxprice = obj.maxNlc  
            else:
                """No rule found, use saholic min nlc as max and min R-Nlc"""
                if identifier is not None:
                    it = Item.query.filter_by(catalog_item_id=identifier).first()
                    try:
                        vendorPricing = inventory_client.getAllItemPricing(it.id)
                        min_nlc = min(pricing.nlc for pricing in vendorPricing)
                        obj.maxNlc = min_nlc
                        obj.minNlc = min_nlc
                        if obj.maxprice == 0:
                            obj.maxprice = obj.maxNlc   
                    except:
                        pass
        populated = populated + 300
        time.sleep(10) 

def addManualDealsInfo():
    for sku in LATEST_UPDATED_ITEMS:
        manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':sku.source_id, 'sku':sku._id}))
        if len(manualDeal) > 0:
            sku.dealType = manualDeal[0]['dealType']
    
    """Remove deal flag from expired deals"""
    manualDeals = list(get_mongo_connection().Catalog.Deals.find({'dealType':1}))
    for manualDeal in manualDeals:
        d_manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':manualDeal['source_id'], 'sku':manualDeal['_id']}))
        if len(d_manualDeal) > 0:
            continue
        else:
            get_mongo_connection().Catalog.Deals.update({'_id':manualDeal['_id']},{"$set":{'dealType':0}},upsert=False, multi=False)

def calculateNlcPoints():
    global LATEST_UPDATED_ITEMS
    print "inside nlc oints"
    for sku in LATEST_UPDATED_ITEMS:
        if sku.maxNlc and sku.minNlc:
            print sku._id
            
            """Create map - TODO"""
            
            if sku.status == 2:
                eolWeight = .60
            else:
                eolWeight = 1.0
            if sku.category_id == 3:
                basePointPercentage = 5.0
                maxNlcPoints = 200
            elif sku.category_id == 5:
                basePointPercentage = 8.0
                maxNlcPoints = 150
            elif sku.category_id == 6:
                basePointPercentage = 5.0
                maxNlcPoints = 200
            
            else:
                basePointPercentage = 10.0
                maxNlcPoints = 150
            discFromMinNlc = float((sku.minNlc - sku.available_price))/sku.available_price *100
            discFromMaxNlc = float((sku.maxNlc - sku.available_price))/sku.available_price *100
            print discFromMinNlc
            print discFromMaxNlc
            if discFromMinNlc > 0:
                nlcPoints = 100/basePointPercentage * discFromMinNlc
            elif discFromMinNlc < 0 and discFromMaxNlc > 0:
                nlcPoints = 0
            else:
                nlcPoints = 100/basePointPercentage * discFromMinNlc
            if (min(nlcPoints,maxNlcPoints)) > 0:
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints)) * eolWeight
            else:
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints))
        else:
            sku.nlcPoints = 0

def commitData():
    global LATEST_UPDATED_ITEMS
    for sku in LATEST_UPDATED_ITEMS:
        #get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{'$set' : sku.__dict__},upsert=True,multi=True)
        get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{"$set":sku.__dict__},upsert=True)
    

def addBestSellerPoints():
    allItems = list(get_mongo_connection().Catalog.Deals.find({})) 
    for sku in allItems:
        bestSellerPoints = list(get_mongo_connection().Catalog.BestSellerPoints.find( {"$and":[{'min_rank': { "$lte": sku['rank'] } }, {'max_rank': { "$gte": sku['rank'] } } , { 'category_id' : sku['category_id'] }, { 'source_id' : sku['source_id'] }] } ))
        if len(bestSellerPoints) > 0:
            print bestSellerPoints[0]['points']
            if (bestSellerPoints[0]['points']) > 0:
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points']) * bestSellerPoints[0]['weightage'] * STATUS_WEIGHTAGE.get(sku['status'])
            else:
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points'])
        else:
            if sku['category_id'] == 6:
                sku['bestSellerPoints'] = 0
            else:
                sku['bestSellerPoints'] = -120
        #sku['totalPoints'] = sku['bestSellerPoints'] + sku['nlcPoints']
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'bestSellerPoints':sku['bestSellerPoints']}},multi=False)
    
    shortageSkus = get_mongo_connection().Catalog.MasterData.find({"$and":[{'is_shortage': 1 }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).distinct('_id')
    print "Shortage skus"
    print shortageSkus

    for sku in allItems:
        deal_item = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':sku['skuBundleId']}).sort('bestSellerPoints',pymongo.DESCENDING).limit(1))
        sku['catalogBestSellerPoints'] = deal_item[0]['bestSellerPoints']
        shortagePoints = 50 if sku['_id'] in shortageSkus else 0
        print "Shortage points for ",sku['_id']
        print shortagePoints
        
        dealPoints = DEAL_POINTS_MAP.get(sku['skuBundleId'])
        
        if dealPoints is not None and dealPoints.manualDealThresholdPrice >= sku['available_price']:
            sku['dealPoints'] = dealPoints.points
            sku['manualDealThresholdPrice'] = dealPoints.manualDealThresholdPrice
        else:
            sku['dealPoints'] = 0
            sku['manualDealThresholdPrice'] = None
            
        
        sku['totalPoints'] = sku['catalogBestSellerPoints'] + sku['nlcPoints'] + shortagePoints + sku['dealPoints'] 
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'catalogBestSellerPoints':sku['catalogBestSellerPoints'],'totalPoints':sku['totalPoints'],'dealPoints':sku['dealPoints'], \
                                                                                    'manualDealThresholdPrice':sku['manualDealThresholdPrice']}},multi=False)
    
    

def populateNegativeDeals():
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
    mc.set("negative_deals", negativeDeals, 600)   

def eliminateSimilarDeals():
    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')
    for skuBundleId in allItems:
        print skuBundleId
        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
        bestPrice = float("inf")
        bestOne = None
        bestSellerPoints = 0
        toUpdate = []
        prepaidBestPrice = float("inf")
        prepaidBestOne = None
        prepaidBestSellerPoints = 0
        for similarItem in similarItems:
            if similarItem['codAvailable'] ==1:
                if mc.get("negative_deals") is None:
                    populateNegativeDeals()
                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                    continue
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                    continue
                if similarItem['available_price'] < bestPrice:
                    bestOne = similarItem
                    bestPrice = similarItem['available_price']
                    bestSellerPoints = similarItem['bestSellerPoints']
                elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
                    bestOne = similarItem
                    bestPrice = similarItem['available_price']
                    bestSellerPoints = similarItem['bestSellerPoints']
                else:
                    pass
            else:
                if mc.get("negative_deals") is None:
                    populateNegativeDeals()
                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                    continue
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                    continue
                if similarItem['source_id'] == SOURCE_MAP.get('PAYTM.COM'):
                    similarItem['available_price'] = similarItem['gross_price'] 
                if similarItem['available_price'] < prepaidBestPrice:
                    prepaidBestOne = similarItem
                    prepaidBestPrice = similarItem['available_price']
                    prepaidBestSellerPoints = similarItem['bestSellerPoints']
                elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
                    prepaidBestOne = similarItem
                    prepaidBestPrice = similarItem['available_price']
                    prepaidBestSellerPoints = similarItem['bestSellerPoints']
                else:
                    pass
        if bestOne is not None or prepaidBestOne is not None:
            for similarItem in similarItems:
                toUpdate.append(similarItem['_id'])
            if bestOne is not None:
                toUpdate.remove(bestOne['_id'])
                get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
            if prepaidBestOne is not None:
                if bestOne is not None:
                    if prepaidBestOne['available_price'] < bestOne['available_price']: 
                        toUpdate.remove(prepaidBestOne['_id'])
                        get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
                else:
                    toUpdate.remove(prepaidBestOne['_id'])
                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
        if len(toUpdate) > 0:
            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

def populateDealPointsBundle():
    global DEAL_POINTS_MAP
    activeDealPoints = get_mongo_connection().Catalog.DealPoints.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})
    for activeDeal in activeDealPoints:
        dealPoints = __DealPoints(activeDeal['dealThresholdPrice'], activeDeal['dealPoints'])
        DEAL_POINTS_MAP[activeDeal['skuBundleId']] = dealPoints 

def main():
    try:
        populateStuff()
        calculateNlc()
        addManualDealsInfo()
    finally:
        session.close()
    calculateNlcPoints()
    commitData()
    populateDealPointsBundle()
    addBestSellerPoints()
    eliminateSimilarDeals()


if __name__=='__main__':
    main()