Subversion Repositories SmartDukaan

import pymongo
from elixir import *
from shop2020.model.v1.catalog.impl import DataService
from shop2020.model.v1.catalog.impl.DataService import Item
from shop2020.clients.InventoryClient import InventoryClient
from dtr.utils.utils import to_java_date, getCashBack
from datetime import datetime, timedelta
import time
import optparse
from dtr.storage.MemCache import MemCache
import traceback
import math
from pymongo import ASCENDING, DESCENDING


parser = optparse.OptionParser()
parser.add_option("-H", "--host", dest="hostname",
                      default="localhost",
                      type="string", help="The HOST where the DB server is running",
                      metavar="host")
parser.add_option("-m", "--m", dest="mongoHost",
                      default="localhost",
                      type="string", help="The HOST where the mongo server is running",
                      metavar="mongo_host")
parser.add_option("-s", "--s", dest="shuffle",
                      default="false",
                      type="string", help="Shuffle ranks for accessories",
                      metavar="shuffle")


(options, args) = parser.parse_args()

mc = MemCache(options.mongoHost)

DataService.initialize(db_hostname=options.hostname)

con = None
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6,'HOMESHOP18.COM':7}
DISCOUNT_TYPE = {'MRP':1,'DP':2}
LATEST_UPDATED_ITEMS = []
STATUS_WEIGHTAGE = {1 : 1.0, 2 : 2.0, 3 : 1.0, 4 : 0.5}
subCatRankList = []

DEAL_POINTS_MAP = {}

now = datetime.now()

DEAL_PRIORITY = [4, 2, 1, 5, 6, 7, 3]  #Increasing order of source deal priority

class __DealPoints:
    
    def __init__(self, manualDealThresholdPrice, points):
        self.manualDealThresholdPrice = manualDealThresholdPrice
        self.points = points
        

class __SkuInfo:
    
    def __init__(self, _id, skuBundleId, category_id, mrp, available_price, source_id, rank, maxNlc, minNlc, schemeAmount, minDiscount, \
                 maxDiscount, discountType, dp, nlcPoints, status, in_stock, maxprice, brand, dealType, brand_id, manualDealThresholdPrice,\
                 codAvailable,showDp,gross_price, subCategoryId,subCategory,shippingCost,netPriceAfterCashBack):
        self._id = _id
        self.skuBundleId = skuBundleId
        self.category_id = category_id
        self.mrp = mrp
        self.available_price = available_price
        self.source_id = source_id
        self.rank = rank
        self.maxNlc = maxNlc
        self.minNlc = minNlc
        self.schemeAmount = schemeAmount
        self.minDiscount = minDiscount
        self.maxDiscount = maxDiscount
        self.discountType = discountType
        self.dp = dp
        self.nlcPoints = nlcPoints
        self.status = status
        self.in_stock = in_stock
        self.maxprice = maxprice
        self.brand = brand
        self.dealType = dealType
        self.brand_id = brand_id
        self.manualDealThresholdPrice = manualDealThresholdPrice
        self.codAvailable = codAvailable
        self.showDp = showDp
        self.gross_price = gross_price
        self.subCategoryId = subCategoryId
        self.subCategory = subCategory
        self.shippingCost = shippingCost
        self.netPriceAfterCashBack = netPriceAfterCashBack


def get_mongo_connection(host=options.mongoHost, port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def populateStuff():
    print "Inside populate"
    global LATEST_UPDATED_ITEMS
    """Fetch latest updated items across portals
       and calculate max and min R-Nlc"""
    offset= 0
    while(True):
        print "Fetching records offset %d and limit %d" %(offset,300)
        topSkus = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'updatedOn': { "$gt": to_java_date(now - timedelta(hours=1))} }, { 'source_id' : { "$in": SOURCE_MAP.values() }},{'category_id':{"$in":[3,5,6]}}] }).skip(offset).limit(300))
        if len((topSkus)) == 0:
            break
        #topSkus = collection.find( {'_id':664})
        for sku in topSkus:
            """Fix this """
            #TODO Compute deal flags else where.
            if sku['source_id'] != SOURCE_MAP.get('PAYTM.COM'):
                netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['available_price'])
            else:
                if sku['codAvailable'] ==0:
                    netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['gross_price'])
                else:
                    netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['available_price'])
            
            info = __SkuInfo(sku['_id'], sku['skuBundleId'], sku['category_id'], sku['mrp'], sku['available_price'], sku['source_id'], sku['rank'], None, None, 0.0, None, \
                             None, None, None, None, sku['status'], sku['in_stock'],sku['maxPrice'],sku['brand'].strip().upper(), 0, sku['brand_id'], None, sku['codAvailable'], 0, sku['gross_price'], sku['subCategoryId'], \
                             sku['subCategory'],sku['shippingCost'],netPriceAfterCashBack)
            exceptionalNlc = list(get_mongo_connection().Catalog.ExceptionalNlc.find( {"$and" : [ {'skuBundleId':info.skuBundleId}, {'overrideNlc':1} ]} ))
            if len(exceptionalNlc) > 0:
                """Exceptional nlc found, no need to calculate max and min R-nlc"""
                info.maxNlc = exceptionalNlc[0]['maxNlc']
                info.minNlc = exceptionalNlc[0]['minNlc']
                if info.maxprice == 0:
                    info.maxprice = exceptionalNlc[0]['maxNlc']
                LATEST_UPDATED_ITEMS.append(info)
                continue
            
            skuSchemeDetails = list(get_mongo_connection().Catalog.SkuSchemeDetails.find( {'skuBundleId':info.skuBundleId}))
            if len(skuSchemeDetails) > 0:
                """Sku scheme details, populate scheme amount (Recently added)"""
                
                #TODO Add start date and end date of scehems
                
                info.schemeAmount = float(skuSchemeDetails[0]['schemeAmount'])
                
            skuDealerPrices = list(get_mongo_connection().Catalog.SkuDealerPrices.find( {'skuBundleId':info.skuBundleId} ) )
            if len(skuDealerPrices) > 0:
                info.dp = skuDealerPrices[0]['dp']
                info.showDp = skuDealerPrices[0]['showDp']
            skuDiscount = list(get_mongo_connection().Catalog.SkuDiscountInfo.find( {'skuBundleId':info.skuBundleId} ) )
            if len(skuDiscount) > 0:
                """Sku rule found, populate max , min Discount and discount type"""
                info.maxDiscount = skuDiscount[0]['max_discount']
                info.minDiscount = skuDiscount[0]['min_discount']
                info.discountType = DISCOUNT_TYPE.get(skuDiscount[0]['discountType'].upper())
                LATEST_UPDATED_ITEMS.append(info)
                continue
            
            categoryDiscount = list(get_mongo_connection().Catalog.CategoryDiscount.find( {"$and" : [{'brand':sku['brand'].strip().upper()}, {'category_id':sku['category_id']} ]} ))
            if len(categoryDiscount) > 0:
                info.maxDiscount = categoryDiscount[0]['max_discount']
                info.minDiscount = categoryDiscount[0]['min_discount']
                info.discountType = DISCOUNT_TYPE.get(categoryDiscount[0]['discountType'].upper())
        
            LATEST_UPDATED_ITEMS.append(info)
        offset = offset + 300
    for lol in LATEST_UPDATED_ITEMS:
        print lol.__dict__
    
    
def calculateNlc():
    global LATEST_UPDATED_ITEMS
    populated = 0
    while(populated <= len(LATEST_UPDATED_ITEMS)):
        inventory_client = InventoryClient().get_client()
        for obj in LATEST_UPDATED_ITEMS[populated:300+populated]:
            if (obj.maxNlc > 0 and obj.minNlc > 0) or obj.category_id==6:
                continue
            saholic_sku = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'skuBundleId': obj.skuBundleId}, { 'source_id' : SOURCE_MAP.get('SAHOLIC')}] }))
            identifier = None
            if len(saholic_sku) > 0:
                identifier = saholic_sku[0]['identifier']
            if obj.discountType == DISCOUNT_TYPE.get('MRP'):
                if obj.mrp == 0:
                    """Now mrp is zero, so we have to use saholic MRP"""
                    if identifier is not None:
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
                        obj.mrp = it.mrp
                if obj.mrp > 0:
                    print obj._id
                    obj.minNlc = obj.mrp - (obj.mrp * obj.maxDiscount/100) - obj.schemeAmount
                    obj.maxNlc = obj.mrp - (obj.mrp * obj.minDiscount/100) - obj.schemeAmount
                    if obj.maxprice == 0:
                        obj.maxprice = obj.maxNlc  
            elif obj.discountType == DISCOUNT_TYPE.get('DP'):
                if obj.dp == 0:
                    """Now dp is zero, so we have to use saholic minimum dp for item"""
                    if identifier is not None:
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
                        try:
                            vendorPricing = inventory_client.getAllItemPricing(it.id)
                            min_dp = min(pricing.dealerPrice for pricing in vendorPricing)
                            obj.dp = min_dp 
                            obj.showDp = 1
                        except:
                            pass
                if obj.dp > 0:
                    obj.minNlc = obj.dp - (obj.dp * obj.maxDiscount/100) - obj.schemeAmount
                    obj.maxNlc = obj.dp - (obj.dp * obj.minDiscount/100) - obj.schemeAmount
                    if obj.maxprice == 0:
                        obj.maxprice = obj.maxNlc  
            else:
                """No rule found, use saholic min nlc as max and min R-Nlc"""
                if identifier is not None:
                    it = Item.query.filter_by(catalog_item_id=identifier).first()
                    try:
                        vendorPricing = inventory_client.getAllItemPricing(it.id)
                        min_nlc = min(pricing.nlc for pricing in vendorPricing)
                        obj.maxNlc = min_nlc
                        obj.minNlc = min_nlc
                        if obj.maxprice == 0:
                            obj.maxprice = obj.maxNlc   
                    except:
                        pass
        populated = populated + 300
        time.sleep(10) 

def addManualDealsInfo():
    for sku in LATEST_UPDATED_ITEMS:
        manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':sku.source_id, 'sku':sku._id}))
        if len(manualDeal) > 0:
            sku.dealType = manualDeal[0]['dealType']
    
    """Remove deal flag from expired deals"""
    manualDeals = list(get_mongo_connection().Catalog.Deals.find({'dealType':1}))
    for manualDeal in manualDeals:
        d_manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':manualDeal['source_id'], 'sku':manualDeal['_id']}))
        if len(d_manualDeal) > 0:
            continue
        else:
            get_mongo_connection().Catalog.Deals.update({'_id':manualDeal['_id']},{"$set":{'dealType':0}},upsert=False, multi=False)

def calculateNlcPoints():
    global LATEST_UPDATED_ITEMS
    print "inside nlc oints"
    for sku in LATEST_UPDATED_ITEMS:
        if sku.maxNlc and sku.minNlc and sku.category_id!=6:
            print sku._id
            
            """Create map - TODO"""
            
            if sku.status == 2:
                eolWeight = .60
            else:
                eolWeight = 1.0
            if sku.category_id == 3:
                basePointPercentage = 5.0
                maxNlcPoints = 200
            elif sku.category_id == 5:
                basePointPercentage = 8.0
                maxNlcPoints = 150
            elif sku.category_id == 6:
                basePointPercentage = 5.0
                maxNlcPoints = 200
            
            else:
                basePointPercentage = 10.0
                maxNlcPoints = 150
            discFromMinNlc = float((sku.minNlc - sku.available_price))/sku.available_price *100
            discFromMaxNlc = float((sku.maxNlc - sku.available_price))/sku.available_price *100
            print discFromMinNlc
            print discFromMaxNlc
            if discFromMinNlc > 0:
                nlcPoints = 100/basePointPercentage * discFromMinNlc
            elif discFromMinNlc < 0 and discFromMaxNlc > 0:
                nlcPoints = 0
            else:
                nlcPoints = 100/basePointPercentage * discFromMinNlc
            if (min(nlcPoints,maxNlcPoints)) > 0:
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints)) * eolWeight
            else:
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints))
        else:
            sku.nlcPoints = 0

def commitData():
    global LATEST_UPDATED_ITEMS
    for sku in LATEST_UPDATED_ITEMS:
        #get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{'$set' : sku.__dict__},upsert=True,multi=True)
        get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{"$set":sku.__dict__},upsert=True)
    

def addBestSellerPoints():
    allItems = list(get_mongo_connection().Catalog.Deals.find({})) 
    for sku in allItems:
        bestSellerPoints = list(get_mongo_connection().Catalog.BestSellerPoints.find( {"$and":[{'min_rank': { "$lte": sku['rank'] } }, {'max_rank': { "$gte": sku['rank'] } } , { 'category_id' : sku['category_id'] }, { 'source_id' : sku['source_id'] }] } ))
        if len(bestSellerPoints) > 0:
            print bestSellerPoints[0]['points']
            if (bestSellerPoints[0]['points']) > 0:
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points']) * bestSellerPoints[0]['weightage'] * STATUS_WEIGHTAGE.get(sku['status'])
            else:
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points'])
        else:
            if sku['category_id'] == 6:
                sku['bestSellerPoints'] = 0
            else:
                sku['bestSellerPoints'] = -120
        #sku['totalPoints'] = sku['bestSellerPoints'] + sku['nlcPoints']
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'bestSellerPoints':sku['bestSellerPoints']}},multi=False)
    
    shortageSkus = get_mongo_connection().Catalog.MasterData.find({"$and":[{'is_shortage': 1 }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).distinct('_id')

    for sku in allItems:
        deal_item = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':sku['skuBundleId']}).sort('bestSellerPoints',pymongo.DESCENDING).limit(1))
        sku['catalogBestSellerPoints'] = deal_item[0]['bestSellerPoints']
        shortagePoints = 50 if sku['_id'] in shortageSkus else 0
        
        dealPoints = DEAL_POINTS_MAP.get(sku['skuBundleId'])
        
        if dealPoints is not None and dealPoints.manualDealThresholdPrice >= sku['available_price']:
            sku['dealPoints'] = dealPoints.points
            sku['manualDealThresholdPrice'] = dealPoints.manualDealThresholdPrice
        else:
            sku['dealPoints'] = 0
            sku['manualDealThresholdPrice'] = None
            
        
        sku['totalPoints'] = sku['catalogBestSellerPoints'] + sku['nlcPoints'] + shortagePoints + sku['dealPoints'] 
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'catalogBestSellerPoints':sku['catalogBestSellerPoints'],'totalPoints':sku['totalPoints'],'dealPoints':sku['dealPoints'], \
                                                                                    'manualDealThresholdPrice':sku['manualDealThresholdPrice']}},multi=False)
    
    

def populateNegativeDeals():
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
    mc.set("negative_deals", negativeDeals, 600)   

def eliminateSimilarDeals():
    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')
    for skuBundleId in allItems:
        print skuBundleId
        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))
        bestPrice = float("inf")
        bestOne = None
        toUpdate = []
        prepaidBestPrice = float("inf")
        prepaidBestOne = None
        for similarItem in similarItems:
            if similarItem['codAvailable'] ==1:
                if mc.get("negative_deals") is None:
                    populateNegativeDeals()
                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                    continue
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                    continue
                if similarItem['netPriceAfterCashBack'] < bestPrice:
                    
                    bestOne = similarItem
                    bestPrice = similarItem['netPriceAfterCashBack']
                elif similarItem['netPriceAfterCashBack'] == bestPrice:
                    
                    try:
                        if (DEAL_PRIORITY.index(int(similarItem['source_id'])) > DEAL_PRIORITY.index(int(bestOne['source_id']))):
                            continue
                    except:
                        traceback.print_exc()
                    
                    bestOne = similarItem
                    bestPrice = similarItem['netPriceAfterCashBack']
                else:
                    pass
            else:
                if mc.get("negative_deals") is None:
                    populateNegativeDeals()
                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                    continue
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                    continue
                if similarItem['netPriceAfterCashBack'] < prepaidBestPrice:
                    prepaidBestOne = similarItem
                    prepaidBestPrice = similarItem['netPriceAfterCashBack']
                elif similarItem['netPriceAfterCashBack'] == prepaidBestPrice:
                    
                    try:
                        if (DEAL_PRIORITY.index(int(similarItem['source_id'])) > DEAL_PRIORITY.index(int(prepaidBestOne['source_id']))):
                            continue
                    except:
                        traceback.print_exc()
                        
                    prepaidBestOne = similarItem
                    prepaidBestPrice = similarItem['netPriceAfterCashBack']
                else:
                    pass
        if bestOne is not None or prepaidBestOne is not None:
            for similarItem in similarItems:
                toUpdate.append(similarItem['_id'])
            if bestOne is not None:
                toUpdate.remove(bestOne['_id'])
                get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
            if prepaidBestOne is not None:
                if bestOne is not None:
                    if prepaidBestOne['netPriceAfterCashBack'] < bestOne['netPriceAfterCashBack']: 
                        toUpdate.remove(prepaidBestOne['_id'])
                        get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
                else:
                    toUpdate.remove(prepaidBestOne['_id'])
                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
        if len(toUpdate) > 0:
            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

def populateDealPointsBundle():
    global DEAL_POINTS_MAP
    activeDealPoints = get_mongo_connection().Catalog.DealPoints.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})
    for activeDeal in activeDealPoints:
        dealPoints = __DealPoints(activeDeal['dealThresholdPrice'], activeDeal['dealPoints'])
        DEAL_POINTS_MAP[activeDeal['skuBundleId']] = dealPoints

def getNetPriceForItem(itemId, source_id, category_id ,price):
    cash_back_type = 0
    cash_back = 0
    try:
        cashBack = getCashBack(itemId, source_id, category_id, mc, options.mongoHost)
        if not cashBack or cashBack.get('cash_back_status')!=1:
            cash_back_type = 0
            cash_back = 0 
            
        else:
            if cashBack['cash_back_type'] in (1,2):
                
                if cashBack.get('maxCashBack') is not None:
                    
                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*price)/100 > cashBack.get('maxCashBack'):
                        cashBack['cash_back_type'] = 2
                        cashBack['cash_back'] = cashBack['maxCashBack']
                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):
                        cashBack['cash_back'] = cashBack['maxCashBack']
                    else:
                        pass
                
                
                
                cash_back_type = cashBack['cash_back_type']
                cash_back = float(cashBack['cash_back'])
    except Exception as cashBackEx:
        pass
    
    if cash_back_type ==1:
        return (price - math.floor(float(cash_back)*price/100))
    elif cash_back_type ==2:
        return (price - cash_back)
    else:
        return price


def getSubCategoryRanking():
    subCatRanks = get_mongo_connection().Catalog.SubCategoriesRank.find().sort([('rank',ASCENDING)])
    for subCatRank in subCatRanks:
        subCatRankList.append(subCatRank['subCategoryId'])
    
def startBucketing():
    
    """Lets create initial buckets of 25 products each from all valid subCategories"""
    initialMap = {}
    for subCat in subCatRankList:
        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':{"$gt":0}}).sort([('internalRank',ASCENDING)]).limit(25))
        temp_list = []
        for product in products:
            if product['skuBundleId'] not in temp_list:
                temp_list.append(product['skuBundleId'])
        initialMap[subCat] = temp_list
    laterMap = {}
    for subCat in subCatRankList:
        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':{"$gt":0}}).sort([('internalRank',ASCENDING)]).skip(25))
        temp_list = []
        for product in products:
            if product['skuBundleId'] not in temp_list:
                temp_list.append(product['skuBundleId'])
        laterMap[subCat] = temp_list
    
    for subCat in subCatRankList:
        no_of_products = len(initialMap.get(subCat))
        if no_of_products < 25:
            toFetch = 25 - no_of_products
            products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':0}).limit(toFetch))
            for product in products:
                initialMap.get(subCat).append(product['skuBundleId'])
            products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':0}).skip(toFetch))
            for product in products:
                laterMap.get(subCat).append(product['skuBundleId'])
        else:
            products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':0}))
            for product in products:
                laterMap.get(subCat).append(product['skuBundleId'])

    """Now we have created two diff. maps having product sorted by rank.Lets create buckets (5 products) from initialMap and (10 products) from laterMap"""
    
    dealRankPoints = 0
    for k, v in initialMap.iteritems():
        dealRankPoints = dealRankPoints + len(v)
    for k, v in laterMap.iteritems():
        dealRankPoints = dealRankPoints + len(v)
    
    while(len(initialMap.keys())>0):
        for subCat in subCatRankList:
            v = initialMap.get(subCat)
            if v is None or len(v) ==0:
                initialMap.pop(subCat, None)
                continue
            get_mongo_connection().Catalog.Deals.update({'skuBundleId':v[0]},{"$set":{'dealRankPoints':dealRankPoints}},upsert=False,multi=True)
            dealRankPoints = dealRankPoints-1
            v.pop(0)
            if len(v) == 0:
                initialMap.pop(subCat, None)
    
    
    print "============================================================"
    
    while(len(laterMap.keys())>0):
        for subCat in subCatRankList:
            v = laterMap.get(subCat)
            if v is None or len(v) ==0:
                laterMap.pop(subCat, None)
                continue
            get_mongo_connection().Catalog.Deals.update({'skuBundleId':v[0]},{"$set":{'dealRankPoints':dealRankPoints}},upsert=False,multi=True)
            dealRankPoints = dealRankPoints-1
            v.pop(0)
            if len(v) == 0:
                laterMap.pop(subCat, None)

def shuffle():
    for subCat in subCatRankList:
        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':0},{'skuBundleId':1}))
        if len(products) >0:
            max_rank = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat}).sort([('internalRank',DESCENDING)]).limit(1))
            rank_to_assign = max_rank[0]['internalRank'] + 1
            for p in products:
                get_mongo_connection().Catalog.MasterData.update({'skuBundleId':p['skuBundleId']},{"$set":{'internalRank':rank_to_assign}},upsert=False,multi=True)
                rank_to_assign = rank_to_assign + 1
        
        new_rank =1
        start = 0
        end = 5
        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':{"$gt":0}},{'skuBundleId':1}).sort([('internalRank',ASCENDING)]).limit(25))
        while(True):
            temp_list = products[start:end]
            shuffled_list = temp_list[1:] + temp_list[:1]
            if len(shuffled_list) > 0:
                for p in shuffled_list:
                    get_mongo_connection().Catalog.MasterData.update({'skuBundleId':p['skuBundleId']},{"$set":{'internalRank':new_rank}},upsert=False,multi=True)
                    new_rank = new_rank + 1
                start = end
                end = end + 5
            else:
                break
        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':{"$gt":0}},{'skuBundleId':1}).sort([('internalRank',ASCENDING)]).skip(25))
        start = 0
        end = 10
        while(True):
            temp_list = products[start:end]
            shuffled_list = temp_list[1:] + temp_list[:1]
            if len(shuffled_list) > 0:
                for p in shuffled_list:
                    get_mongo_connection().Catalog.MasterData.update({'skuBundleId':p['skuBundleId']},{"$set":{'internalRank':new_rank}},upsert=False,multi=True)
                    new_rank = new_rank + 1
                start = end
                end = end + 10
            else:
                break
              
def main():
    try:
        print "Starting populating stuff ", datetime.now()
        populateStuff()
        print "Starting calculate nlc stuff ", datetime.now()
        calculateNlc()
        print "Starting adding manualdeals stuff ", datetime.now()
        addManualDealsInfo()
    finally:
        session.close()
    print "Starting calculate nlc points ", datetime.now()
    calculateNlcPoints()
    print "commiting data ", datetime.now()
    commitData()
    print "Populate deal points bundle ", datetime.now()
    populateDealPointsBundle()
    print "Add best seller points ", datetime.now()
    addBestSellerPoints()
    print "eliminate similar deals ", datetime.now()
    eliminateSimilarDeals()
    print "done ", datetime.now()
    getSubCategoryRanking()
    if options.shuffle=='true':
        print "Shuffling products"
        shuffle()
    startBucketing()
    print "Done with bucketing",datetime.now()
    


if __name__=='__main__':
    main()