Subversion Repositories SmartDukaan

import pymongo

from elixir import *

from shop2020.model.v1.catalog.impl import DataService

from shop2020.model.v1.catalog.impl.DataService import Item

from shop2020.clients.InventoryClient import InventoryClient

from dtr.utils.utils import to_java_date, getCashBack

from datetime import datetime, timedelta

import time

import optparse

from dtr.storage.MemCache import MemCache

import traceback

import math

from pymongo import ASCENDING, DESCENDING

parser = optparse.OptionParser()

parser.add_option("-H", "--host", dest="hostname",

                      default="localhost",

                      type="string", help="The HOST where the DB server is running",

                      metavar="host")

parser.add_option("-m", "--m", dest="mongoHost",

                      default="localhost",

                      type="string", help="The HOST where the mongo server is running",

                      metavar="mongo_host")

parser.add_option("-s", "--s", dest="shuffle",

                      default="false",

                      type="string", help="Shuffle ranks for accessories",

                      metavar="shuffle")

(options, args) = parser.parse_args()

mc = MemCache(options.mongoHost)

DataService.initialize(db_hostname=options.hostname)

con = None

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6,'HOMESHOP18.COM':7}

DISCOUNT_TYPE = {'MRP':1,'DP':2}

LATEST_UPDATED_ITEMS = []

STATUS_WEIGHTAGE = {1 : 1.0, 2 : 2.0, 3 : 1.0, 4 : 0.5}

subCatRankList = []

DEAL_POINTS_MAP = {}

now = datetime.now()

DEAL_PRIORITY = [4, 2, 1, 5, 6, 7, 3]  #Increasing order of source deal priority

class __DealPoints:

    def __init__(self, manualDealThresholdPrice, points):

        self.manualDealThresholdPrice = manualDealThresholdPrice

        self.points = points

class __SkuInfo:

    def __init__(self, _id, skuBundleId, category_id, mrp, available_price, source_id, rank, maxNlc, minNlc, schemeAmount, minDiscount, \

                 maxDiscount, discountType, dp, nlcPoints, status, in_stock, maxprice, brand, dealType, brand_id, manualDealThresholdPrice,\

                 codAvailable,showDp,gross_price, subCategoryId,subCategory,shippingCost,netPriceAfterCashBack, exclusive):

        self._id = _id

        self.skuBundleId = skuBundleId

        self.category_id = category_id

        self.mrp = mrp

        self.available_price = available_price

        self.source_id = source_id

        self.rank = rank

        self.maxNlc = maxNlc

        self.minNlc = minNlc

        self.schemeAmount = schemeAmount

        self.minDiscount = minDiscount

        self.maxDiscount = maxDiscount

        self.discountType = discountType

        self.dp = dp

        self.nlcPoints = nlcPoints

        self.status = status

        self.in_stock = in_stock

        self.maxprice = maxprice

        self.brand = brand

        self.dealType = dealType

        self.brand_id = brand_id

        self.manualDealThresholdPrice = manualDealThresholdPrice

        self.codAvailable = codAvailable

        self.showDp = showDp

        self.gross_price = gross_price

        self.subCategoryId = subCategoryId

        self.subCategory = subCategory

        self.shippingCost = shippingCost

        self.netPriceAfterCashBack = netPriceAfterCashBack

        self.exclusive = exclusive

def get_mongo_connection(host=options.mongoHost, port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def populateStuff():

    print "Inside populate"

    global LATEST_UPDATED_ITEMS

    """Fetch latest updated items across portals

       and calculate max and min R-Nlc"""

    offset= 0

    while(True):

        print "Fetching records offset %d and limit %d" %(offset,300)

        topSkus = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'updatedOn': { "$gt": to_java_date(now - timedelta(hours=1))} }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).skip(offset).limit(300))

        if len((topSkus)) == 0:

            break

        #topSkus = collection.find( {'_id':664})

        for sku in topSkus:

            """Fix this """

            #TODO Compute deal flags else where.

            if sku['source_id'] != SOURCE_MAP.get('PAYTM.COM'):

                netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['available_price'])

            else:

                if sku['codAvailable'] ==0:

                    netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['gross_price'])

                else:

                    netPriceAfterCashBack = getNetPriceForItem(sku['_id'], sku['source_id'], sku['category_id'], sku['available_price'])

            info = __SkuInfo(sku['_id'], sku['skuBundleId'], sku['category_id'], sku['mrp'], sku['available_price'], sku['source_id'], sku['rank'], None, None, 0.0, None, \

                             None, None, None, None, sku['status'], sku['in_stock'],sku['maxPrice'],sku['brand'].strip().upper(), 0, sku['brand_id'], None, sku['codAvailable'], 0, sku['gross_price'], sku['subCategoryId'], \

                             sku['subCategory'],sku['shippingCost'],netPriceAfterCashBack, sku['exclusive'])

            exceptionalNlc = list(get_mongo_connection().Catalog.ExceptionalNlc.find( {"$and" : [ {'skuBundleId':info.skuBundleId}, {'overrideNlc':1} ]} ))

            if len(exceptionalNlc) > 0:

                """Exceptional nlc found, no need to calculate max and min R-nlc"""

                info.maxNlc = exceptionalNlc[0]['maxNlc']

                info.minNlc = exceptionalNlc[0]['minNlc']

                if info.maxprice == 0:

                    info.maxprice = exceptionalNlc[0]['maxNlc']

                LATEST_UPDATED_ITEMS.append(info)

                continue

            skuSchemeDetails = list(get_mongo_connection().Catalog.SkuSchemeDetails.find( {'skuBundleId':info.skuBundleId}))

            if len(skuSchemeDetails) > 0:

                """Sku scheme details, populate scheme amount (Recently added)"""

                #TODO Add start date and end date of scehems

                info.schemeAmount = float(skuSchemeDetails[0]['schemeAmount'])

            skuDealerPrices = list(get_mongo_connection().Catalog.SkuDealerPrices.find( {'skuBundleId':info.skuBundleId} ) )

            if len(skuDealerPrices) > 0:

                info.dp = skuDealerPrices[0]['dp']

                info.showDp = skuDealerPrices[0]['showDp']

            skuDiscount = list(get_mongo_connection().Catalog.SkuDiscountInfo.find( {'skuBundleId':info.skuBundleId} ) )

            if len(skuDiscount) > 0:

                """Sku rule found, populate max , min Discount and discount type"""

                info.maxDiscount = skuDiscount[0]['max_discount']

                info.minDiscount = skuDiscount[0]['min_discount']

                info.discountType = DISCOUNT_TYPE.get(skuDiscount[0]['discountType'].upper())

                LATEST_UPDATED_ITEMS.append(info)

                continue

            categoryDiscount = list(get_mongo_connection().Catalog.CategoryDiscount.find( {"$and" : [{'brand':sku['brand'].strip().upper()}, {'category_id':sku['category_id']} ]} ))

            if len(categoryDiscount) > 0:

                info.maxDiscount = categoryDiscount[0]['max_discount']

                info.minDiscount = categoryDiscount[0]['min_discount']

                info.discountType = DISCOUNT_TYPE.get(categoryDiscount[0]['discountType'].upper())

            LATEST_UPDATED_ITEMS.append(info)

        offset = offset + 300

    for lol in LATEST_UPDATED_ITEMS:

        print lol.__dict__

def calculateNlc():

    global LATEST_UPDATED_ITEMS

    populated = 0

    while(populated <= len(LATEST_UPDATED_ITEMS)):

        inventory_client = InventoryClient().get_client()

        for obj in LATEST_UPDATED_ITEMS[populated:300+populated]:

            if (obj.maxNlc > 0 and obj.minNlc > 0) or obj.category_id==6:

                continue

            saholic_sku = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'skuBundleId': obj.skuBundleId}, { 'source_id' : SOURCE_MAP.get('SAHOLIC')}] }))

            identifier = None

            if len(saholic_sku) > 0:

                identifier = saholic_sku[0]['identifier']

            if obj.discountType == DISCOUNT_TYPE.get('MRP'):

                if obj.mrp == 0:

                    """Now mrp is zero, so we have to use saholic MRP"""

                    if identifier is not None:

                        it = Item.query.filter_by(catalog_item_id=identifier).first()

                        obj.mrp = it.mrp

                if obj.mrp > 0:

                    print obj._id

                    obj.minNlc = obj.mrp - (obj.mrp * obj.maxDiscount/100) - obj.schemeAmount

                    obj.maxNlc = obj.mrp - (obj.mrp * obj.minDiscount/100) - obj.schemeAmount

                    if obj.maxprice == 0:

                        obj.maxprice = obj.maxNlc  

            elif obj.discountType == DISCOUNT_TYPE.get('DP'):

                if obj.dp == 0:

                    """Now dp is zero, so we have to use saholic minimum dp for item"""

                    if identifier is not None:

                        it = Item.query.filter_by(catalog_item_id=identifier).first()

                        try:

                            vendorPricing = inventory_client.getAllItemPricing(it.id)

                            min_dp = min(pricing.dealerPrice for pricing in vendorPricing)

                            obj.dp = min_dp 

                            obj.showDp = 1

                        except:

                            pass

                if obj.dp > 0:

                    obj.minNlc = obj.dp - (obj.dp * obj.maxDiscount/100) - obj.schemeAmount

                    obj.maxNlc = obj.dp - (obj.dp * obj.minDiscount/100) - obj.schemeAmount

                    if obj.maxprice == 0:

                        obj.maxprice = obj.maxNlc  

            else:

                """No rule found, use saholic min nlc as max and min R-Nlc"""

                if identifier is not None:

                    it = Item.query.filter_by(catalog_item_id=identifier).first()

                    try:

                        vendorPricing = inventory_client.getAllItemPricing(it.id)

                        min_nlc = min(pricing.nlc for pricing in vendorPricing)

                        obj.maxNlc = min_nlc

                        obj.minNlc = min_nlc

                        if obj.maxprice == 0:

                            obj.maxprice = obj.maxNlc   

                    except:

                        pass

        populated = populated + 300

        time.sleep(10) 

def addManualDealsInfo():

    for sku in LATEST_UPDATED_ITEMS:

        manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':sku.source_id, 'sku':sku._id}))

        if len(manualDeal) > 0:

            sku.dealType = manualDeal[0]['dealType']

    """Remove deal flag from expired deals"""

    manualDeals = list(get_mongo_connection().Catalog.Deals.find({'dealType':1}))

    for manualDeal in manualDeals:

        d_manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':manualDeal['source_id'], 'sku':manualDeal['_id']}))

        if len(d_manualDeal) > 0:

            continue

        else:

            get_mongo_connection().Catalog.Deals.update({'_id':manualDeal['_id']},{"$set":{'dealType':0}},upsert=False, multi=False)

def calculateNlcPoints():

    global LATEST_UPDATED_ITEMS

    print "inside nlc oints"

    for sku in LATEST_UPDATED_ITEMS:

        if sku.maxNlc and sku.minNlc and sku.category_id!=6:

            print sku._id

            """Create map - TODO"""

            if sku.status == 2:

                eolWeight = .60

            else:

                eolWeight = 1.0

            if sku.category_id == 3:

                basePointPercentage = 5.0

                maxNlcPoints = 200

            elif sku.category_id == 5:

                basePointPercentage = 8.0

                maxNlcPoints = 150

            elif sku.category_id == 6:

                basePointPercentage = 5.0

                maxNlcPoints = 200

            else:

                basePointPercentage = 10.0

                maxNlcPoints = 150

            discFromMinNlc = float((sku.minNlc - sku.available_price))/sku.available_price *100

            discFromMaxNlc = float((sku.maxNlc - sku.available_price))/sku.available_price *100

            print discFromMinNlc

            print discFromMaxNlc

            if discFromMinNlc > 0:

                nlcPoints = 100/basePointPercentage * discFromMinNlc

            elif discFromMinNlc < 0 and discFromMaxNlc > 0:

                nlcPoints = 0

            else:

                nlcPoints = 100/basePointPercentage * discFromMinNlc

            if (min(nlcPoints,maxNlcPoints)) > 0:

                sku.nlcPoints = (min(nlcPoints,maxNlcPoints)) * eolWeight

            else:

                sku.nlcPoints = (min(nlcPoints,maxNlcPoints))

        else:

            sku.nlcPoints = 0

def commitData():

    global LATEST_UPDATED_ITEMS

    for sku in LATEST_UPDATED_ITEMS:

        #get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{'$set' : sku.__dict__},upsert=True,multi=True)

        get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{"$set":sku.__dict__},upsert=True)

def addBestSellerPoints():

    allItems = list(get_mongo_connection().Catalog.Deals.find({})) 

    for sku in allItems:

        bestSellerPoints = list(get_mongo_connection().Catalog.BestSellerPoints.find( {"$and":[{'min_rank': { "$lte": sku['rank'] } }, {'max_rank': { "$gte": sku['rank'] } } , { 'category_id' : sku['category_id'] }, { 'source_id' : sku['source_id'] }] } ))

        if len(bestSellerPoints) > 0:

            print bestSellerPoints[0]['points']

            if (bestSellerPoints[0]['points']) > 0:

                sku['bestSellerPoints'] = (bestSellerPoints[0]['points']) * bestSellerPoints[0]['weightage'] * STATUS_WEIGHTAGE.get(sku['status'])

            else:

                sku['bestSellerPoints'] = (bestSellerPoints[0]['points'])

        else:

            if sku['category_id'] == 6:

                sku['bestSellerPoints'] = 0

            else:

                sku['bestSellerPoints'] = -120

        #sku['totalPoints'] = sku['bestSellerPoints'] + sku['nlcPoints']

        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'bestSellerPoints':sku['bestSellerPoints']}},multi=False)

    shortageSkus = get_mongo_connection().Catalog.MasterData.find({"$and":[{'is_shortage': 1 }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).distinct('_id')

    for sku in allItems:

        deal_item = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':sku['skuBundleId']}).sort('bestSellerPoints',pymongo.DESCENDING).limit(1))

        sku['catalogBestSellerPoints'] = deal_item[0]['bestSellerPoints']

        shortagePoints = 50 if sku['_id'] in shortageSkus else 0

        dealPoints = DEAL_POINTS_MAP.get(sku['skuBundleId'])

        if dealPoints is not None and dealPoints.manualDealThresholdPrice >= sku['available_price']:

            sku['dealPoints'] = dealPoints.points

            sku['manualDealThresholdPrice'] = dealPoints.manualDealThresholdPrice

        else:

            sku['dealPoints'] = 0

            sku['manualDealThresholdPrice'] = None

        sku['totalPoints'] = sku['catalogBestSellerPoints'] + sku['nlcPoints'] + shortagePoints + sku['dealPoints'] 

        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'catalogBestSellerPoints':sku['catalogBestSellerPoints'],'totalPoints':sku['totalPoints'],'dealPoints':sku['dealPoints'], \

                                                                                    'manualDealThresholdPrice':sku['manualDealThresholdPrice']}},multi=False)

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)   

def eliminateSimilarDeals():

    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')

    for skuBundleId in allItems:

        print skuBundleId

        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))

        bestPrice = float("inf")

        bestOne = None

        toUpdate = []

        prepaidBestPrice = float("inf")

        prepaidBestOne = None

        for similarItem in similarItems:

            if similarItem['codAvailable'] ==1:

                if mc.get("negative_deals") is None:

                    populateNegativeDeals()

                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                    continue

                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:

                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                    continue

                if similarItem['netPriceAfterCashBack'] < bestPrice:

                    bestOne = similarItem

                    bestPrice = similarItem['netPriceAfterCashBack']

                elif similarItem['netPriceAfterCashBack'] == bestPrice:

                    try:

                        if (DEAL_PRIORITY.index(int(similarItem['source_id'])) > DEAL_PRIORITY.index(int(bestOne['source_id']))):

                            continue

                    except:

                        traceback.print_exc()

                    bestOne = similarItem

                    bestPrice = similarItem['netPriceAfterCashBack']

                else:

                    pass

            else:

                if mc.get("negative_deals") is None:

                    populateNegativeDeals()

                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):

                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                    continue

                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0 and similarItem['category_id']!=6:

                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                    continue

                if similarItem['netPriceAfterCashBack'] < prepaidBestPrice:

                    prepaidBestOne = similarItem

                    prepaidBestPrice = similarItem['netPriceAfterCashBack']

                elif similarItem['netPriceAfterCashBack'] == prepaidBestPrice:

                    try:

                        if (DEAL_PRIORITY.index(int(similarItem['source_id'])) > DEAL_PRIORITY.index(int(prepaidBestOne['source_id']))):

                            continue

                    except:

                        traceback.print_exc()

                    prepaidBestOne = similarItem

                    prepaidBestPrice = similarItem['netPriceAfterCashBack']

                else:

                    pass

        if bestOne is not None or prepaidBestOne is not None:

            for similarItem in similarItems:

                toUpdate.append(similarItem['_id'])

            if bestOne is not None:

                toUpdate.remove(bestOne['_id'])

                get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})

            if prepaidBestOne is not None:

                if bestOne is not None:

                    if prepaidBestOne['netPriceAfterCashBack'] < bestOne['netPriceAfterCashBack']: 

                        toUpdate.remove(prepaidBestOne['_id'])

                        get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

                else:

                    toUpdate.remove(prepaidBestOne['_id'])

                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

        if len(toUpdate) > 0:

            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

def populateDealPointsBundle():

    global DEAL_POINTS_MAP

    activeDealPoints = get_mongo_connection().Catalog.DealPoints.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})

    for activeDeal in activeDealPoints:

        dealPoints = __DealPoints(activeDeal['dealThresholdPrice'], activeDeal['dealPoints'])

        DEAL_POINTS_MAP[activeDeal['skuBundleId']] = dealPoints

def getNetPriceForItem(itemId, source_id, category_id ,price):

    cash_back_type = 0

    cash_back = 0

    try:

        cashBack = getCashBack(itemId, source_id, category_id, mc, options.mongoHost)

        if not cashBack or cashBack.get('cash_back_status')!=1:

            cash_back_type = 0

            cash_back = 0 

        else:

            if cashBack['cash_back_type'] in (1,2):

                if cashBack.get('maxCashBack') is not None:

                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*price)/100 > cashBack.get('maxCashBack'):

                        cashBack['cash_back_type'] = 2

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):

                        cashBack['cash_back'] = cashBack['maxCashBack']

                    else:

                        pass

                cash_back_type = cashBack['cash_back_type']

                cash_back = float(cashBack['cash_back'])

    except Exception as cashBackEx:

        pass

    if cash_back_type ==1:

        return (price - math.floor(float(cash_back)*price/100))

    elif cash_back_type ==2:

        return (price - cash_back)

    else:

        return price

def getSubCategoryRanking():

    subCatRanks = get_mongo_connection().Catalog.SubCategoriesRank.find().sort([('rank',ASCENDING)])

    for subCatRank in subCatRanks:

        subCatRankList.append(subCatRank['subCategoryId'])

def startBucketing():

    """Lets create initial buckets of 25 products each from all valid subCategories"""

    initialMap = {}

    for subCat in subCatRankList:

        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':{"$gt":0}}).sort([('internalRank',ASCENDING)]).limit(25))

        temp_list = []

        for product in products:

            if product['skuBundleId'] not in temp_list:

                temp_list.append(product['skuBundleId'])

        initialMap[subCat] = temp_list

    laterMap = {}

    for subCat in subCatRankList:

        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':{"$gt":0}}).sort([('internalRank',ASCENDING)]).skip(25))

        temp_list = []

        for product in products:

            if product['skuBundleId'] not in temp_list:

                temp_list.append(product['skuBundleId'])

        laterMap[subCat] = temp_list

    for subCat in subCatRankList:

        no_of_products = len(initialMap.get(subCat))

        if no_of_products < 25:

            toFetch = 25 - no_of_products

            products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':0}).limit(toFetch))

            for product in products:

                initialMap.get(subCat).append(product['skuBundleId'])

            products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':0}).skip(toFetch))

            for product in products:

                laterMap.get(subCat).append(product['skuBundleId'])

        else:

            products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':0}))

            for product in products:

                laterMap.get(subCat).append(product['skuBundleId'])

    """Now we have created two diff. maps having product sorted by rank.Lets create buckets (5 products) from initialMap and (10 products) from laterMap"""

    dealRankPoints = 0

    for k, v in initialMap.iteritems():

        dealRankPoints = dealRankPoints + len(v)

    for k, v in laterMap.iteritems():

        dealRankPoints = dealRankPoints + len(v)

    while(len(initialMap.keys())>0):

        for subCat in subCatRankList:

            v = initialMap.get(subCat)

            if v is None or len(v) ==0:

                initialMap.pop(subCat, None)

                continue

            get_mongo_connection().Catalog.Deals.update({'skuBundleId':v[0]},{"$set":{'dealRankPoints':dealRankPoints}},upsert=False,multi=True)

            dealRankPoints = dealRankPoints-1

            v.pop(0)

            if len(v) == 0:

                initialMap.pop(subCat, None)

    print "============================================================"

    while(len(laterMap.keys())>0):

        for subCat in subCatRankList:

            v = laterMap.get(subCat)

            if v is None or len(v) ==0:

                laterMap.pop(subCat, None)

                continue

            get_mongo_connection().Catalog.Deals.update({'skuBundleId':v[0]},{"$set":{'dealRankPoints':dealRankPoints}},upsert=False,multi=True)

            dealRankPoints = dealRankPoints-1

            v.pop(0)

            if len(v) == 0:

                laterMap.pop(subCat, None)

def shuffle():

    for subCat in subCatRankList:

        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':0},{'skuBundleId':1}))

        if len(products) >0:

            max_rank = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat}).sort([('internalRank',DESCENDING)]).limit(1))

            rank_to_assign = max_rank[0]['internalRank'] + 1

            for p in products:

                get_mongo_connection().Catalog.MasterData.update({'skuBundleId':p['skuBundleId']},{"$set":{'internalRank':rank_to_assign}},upsert=False,multi=True)

                rank_to_assign = rank_to_assign + 1

        new_rank =1

        start = 0

        end = 5

        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':{"$gt":0}},{'skuBundleId':1}).sort([('internalRank',ASCENDING)]).limit(25))

        while(True):

            temp_list = products[start:end]

            shuffled_list = temp_list[1:] + temp_list[:1]

            if len(shuffled_list) > 0:

                for p in shuffled_list:

                    get_mongo_connection().Catalog.MasterData.update({'skuBundleId':p['skuBundleId']},{"$set":{'internalRank':new_rank}},upsert=False,multi=True)

                    new_rank = new_rank + 1

                start = end

                end = end + 5

            else:

                break

        products = list(get_mongo_connection().Catalog.MasterData.find({'subCategoryId':subCat,'internalRank':{"$gt":0}},{'skuBundleId':1}).sort([('internalRank',ASCENDING)]).skip(25))

        start = 0

        end = 10

        while(True):

            temp_list = products[start:end]

            shuffled_list = temp_list[1:] + temp_list[:1]

            if len(shuffled_list) > 0:

                for p in shuffled_list:

                    get_mongo_connection().Catalog.MasterData.update({'skuBundleId':p['skuBundleId']},{"$set":{'internalRank':new_rank}},upsert=False,multi=True)

                    new_rank = new_rank + 1

                start = end

                end = end + 10

            else:

                break

def main():

    try:

        print "Starting populating stuff ", datetime.now()

        populateStuff()

        print "Starting calculate nlc stuff ", datetime.now()

        calculateNlc()

        print "Starting adding manualdeals stuff ", datetime.now()

        addManualDealsInfo()

    finally:

        session.close()

    print "Starting calculate nlc points ", datetime.now()

    calculateNlcPoints()

    print "commiting data ", datetime.now()

    commitData()

    print "Populate deal points bundle ", datetime.now()

    populateDealPointsBundle()

    print "Add best seller points ", datetime.now()

    addBestSellerPoints()

    print "eliminate similar deals ", datetime.now()

    eliminateSimilarDeals()

    print "done ", datetime.now()

    getSubCategoryRanking()

    if options.shuffle=='true':

        print "Shuffling products"

        shuffle()

    startBucketing()

    print "Done with bucketing",datetime.now()

if __name__=='__main__':

    main()