Subversion Repositories SmartDukaan

Rev

Rev 14741 | Rev 14842 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13755 kshitij.so 1 
import pymongo
2 
from elixir import *
3 
from shop2020.model.v1.catalog.impl import DataService
4 
from shop2020.model.v1.catalog.impl.DataService import Item
5 
from shop2020.clients.InventoryClient import InventoryClient
6 
from dtr.utils.utils import to_java_date
7 
from datetime import datetime, timedelta
8 
import time
14258 kshitij.so 9 
import optparse
14325 kshitij.so 10 
from dtr.storage.MemCache import MemCache
14705 kshitij.so 11 
from dtr.utils.utils import getCashBack
13755 kshitij.so 12 
 
14258 kshitij.so 13 
parser = optparse.OptionParser()
14263 kshitij.so 14 
parser.add_option("-H", "--host", dest="hostname",
14258 kshitij.so 15 
                      default="localhost",
16 
                      type="string", help="The HOST where the DB server is running",
17 
                      metavar="host")
18 
parser.add_option("-m", "--m", dest="mongoHost",
19 
                      default="localhost",
20 
                      type="string", help="The HOST where the mongo server is running",
21 
                      metavar="mongo_host")
13755 kshitij.so 22 
 
14258 kshitij.so 23 
(options, args) = parser.parse_args()
14325 kshitij.so 24 
 
25 
mc = MemCache(options.mongoHost)
26 
 
14258 kshitij.so 27 
DataService.initialize(db_hostname=options.hostname)
28 
 
13755 kshitij.so 29 
con = None
30 
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4}
31 
DISCOUNT_TYPE = {'MRP':1,'DP':2}
32 
LATEST_UPDATED_ITEMS = []
33 
 
34 
now = datetime.now()
35 
 
36 
class __SkuInfo:
37 
 
38 
    def __init__(self, _id, skuBundleId, category_id, mrp, available_price, source_id, rank, maxNlc, minNlc, schemeAmount, minDiscount, \
14779 kshitij.so 39 
                 maxDiscount, discountType, dp, nlcPoints, status, in_stock, maxprice, brand, dealType, dealPoints, brand_id):
13755 kshitij.so 40 
        self._id = _id
41 
        self.skuBundleId = skuBundleId
42 
        self.category_id = category_id
43 
        self.mrp = mrp
44 
        self.available_price = available_price
45 
        self.source_id = source_id
46 
        self.rank = rank
47 
        self.maxNlc = maxNlc
48 
        self.minNlc = minNlc
49 
        self.schemeAmount = schemeAmount
50 
        self.minDiscount = minDiscount
51 
        self.maxDiscount = maxDiscount
52 
        self.discountType = discountType
53 
        self.dp = dp
54 
        self.nlcPoints = nlcPoints
13824 kshitij.so 55 
        self.status = status
56 
        self.in_stock = in_stock
57 
        self.maxprice = maxprice
14035 kshitij.so 58 
        self.brand = brand
14306 kshitij.so 59 
        self.dealType = dealType
14388 kshitij.so 60 
        self.dealPoints = dealPoints
14779 kshitij.so 61 
        self.brand_id = brand_id
13755 kshitij.so 62 
 
63 
 
14258 kshitij.so 64 
def get_mongo_connection(host=options.mongoHost, port=27017):
13755 kshitij.so 65 
    global con
66 
    if con is None:
67 
        print "Establishing connection %s host and port %d" %(host,port)
68 
        try:
69 
            con = pymongo.MongoClient(host, port)
70 
        except Exception, e:
71 
            print e
72 
            return None
73 
    return con
74 
 
75 
def populateStuff():
76 
    print "Inside populate"
77 
    global LATEST_UPDATED_ITEMS
78 
    """Fetch latest updated items across portals
79 
       and calculate max and min R-Nlc"""
80 
    offset= 0
81 
    while(True):
82 
        print "Fetching records offset %d and limit %d" %(offset,300)
83 
        topSkus = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'updatedOn': { "$gt": to_java_date(now - timedelta(hours=4))} }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).skip(offset).limit(300))
84 
        if len((topSkus)) == 0:
85 
            break
86 
        #topSkus = collection.find( {'_id':664})
87 
        for sku in topSkus:
13908 kshitij.so 88 
            """Fix this """
89 
            #TODO Compute deal flags else where.
13755 kshitij.so 90 
            info = __SkuInfo(sku['_id'], sku['skuBundleId'], sku['category_id'], sku['mrp'], sku['available_price'], sku['source_id'], sku['rank'], None, None, 0.0, None, \
14779 kshitij.so 91 
                             None, None, None, None, sku['status'], sku['in_stock'],None,sku['brand'].strip().upper(), 0, 0, sku['brand_id'])
13755 kshitij.so 92 
            exceptionalNlc = list(get_mongo_connection().Catalog.ExceptionalNlc.find( {"$and" : [ {'sku':info._id}, {'overrideNlc':1} ]} ))
93 
            if len(exceptionalNlc) > 0:
94 
                """Exceptional nlc found, no need to calculate max and min R-nlc"""
95 
                info.maxNlc = exceptionalNlc[0]['maxNlc']
96 
                info.minNlc = exceptionalNlc[0]['minNlc']
13824 kshitij.so 97 
                info.maxprice = exceptionalNlc[0]['maxNlc']
13755 kshitij.so 98 
                LATEST_UPDATED_ITEMS.append(info)
99 
                continue
100 
 
101 
            skuSchemeDetails = list(get_mongo_connection().Catalog.SkuSchemeDetails.find( {'sku':info._id}).sort([('addedOn',pymongo.DESCENDING)]).limit(1))
102 
            if len(skuSchemeDetails) > 0:
103 
                """Sku scheme details, populate scheme amount (Recently added)"""
13824 kshitij.so 104 
 
105 
                #TODO Add start date and end date of scehems
106 
 
13755 kshitij.so 107 
                info.schemeAmount = float(skuSchemeDetails[0]['schemeAmount'])
108 
 
109 
            skuDealerPrices = list(get_mongo_connection().Catalog.SkuDealerPrices.find( {'sku':info._id} ) )
110 
            if len(skuDealerPrices) > 0:
111 
                info.dp = skuDealerPrices[0]['dp']
112 
            skuDiscount = list(get_mongo_connection().Catalog.SkuDiscountInfo.find( {'sku':info._id} ) )
113 
            if len(skuDiscount) > 0:
114 
                """Sku rule found, populate max , min Discount and discount type"""
115 
                info.maxDiscount = skuDiscount[0]['max_discount']
116 
                info.minDiscount = skuDiscount[0]['min_discount']
117 
                info.discountType = DISCOUNT_TYPE.get(skuDiscount[0]['discountType'].upper())
118 
                LATEST_UPDATED_ITEMS.append(info)
119 
                continue
120 
 
14035 kshitij.so 121 
            categoryDiscount = list(get_mongo_connection().Catalog.CategoryDiscount.find( {"$and" : [{'brand':sku['brand'].strip().upper()}, {'category_id':sku['category_id']} ]} ))
13755 kshitij.so 122 
            if len(categoryDiscount) > 0:
123 
                info.maxDiscount = categoryDiscount[0]['max_discount']
124 
                info.minDiscount = categoryDiscount[0]['min_discount']
125 
                info.discountType = DISCOUNT_TYPE.get(categoryDiscount[0]['discountType'].upper())
126 
 
127 
            LATEST_UPDATED_ITEMS.append(info)
128 
        offset = offset + 300
129 
    for lol in LATEST_UPDATED_ITEMS:
130 
        print lol.__dict__
131 
 
132 
 
133 
def calculateNlc():
134 
    global LATEST_UPDATED_ITEMS
135 
    populated = 0
136 
    while(populated <= len(LATEST_UPDATED_ITEMS)):
137 
        inventory_client = InventoryClient().get_client()
138 
        for obj in LATEST_UPDATED_ITEMS[populated:300+populated]:
139 
            if obj.maxNlc > 0 and obj.minNlc > 0:
140 
                continue
141 
            saholic_sku = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'skuBundleId': obj.skuBundleId}, { 'source_id' : SOURCE_MAP.get('SAHOLIC')}] }))
142 
            identifier = None
143 
            if len(saholic_sku) > 0:
144 
                identifier = saholic_sku[0]['identifier']
145 
            if obj.discountType == DISCOUNT_TYPE.get('MRP'):
146 
                if obj.mrp == 0:
147 
                    """Now mrp is zero, so we have to use saholic MRP"""
148 
                    if identifier is not None:
149 
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
150 
                        obj.mrp = it.mrp
151 
                if obj.mrp > 0:
152 
                    obj.minNlc = obj.mrp - (obj.mrp * obj.maxDiscount/100) - obj.schemeAmount
13824 kshitij.so 153 
                    obj.maxNlc = obj.mrp - (obj.mrp * obj.minDiscount/100) - obj.schemeAmount
13982 kshitij.so 154 
                    obj.maxprice = obj.maxNlc
13755 kshitij.so 155 
            elif obj.discountType == DISCOUNT_TYPE.get('DP'):
156 
                if obj.dp == 0:
157 
                    """Now dp is zero, so we have to use saholic minimum dp for item"""
158 
                    if identifier is not None:
159 
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
160 
                        try:
161 
                            vendorPricing = inventory_client.getAllItemPricing(it.id)
162 
                            min_dp = min(pricing.dealerPrice for pricing in vendorPricing)
163 
                            obj.dp = min_dp
164 
                        except:
165 
                            pass
166 
                if obj.dp > 0:
167 
                    obj.minNlc = obj.dp - (obj.dp * obj.maxDiscount/100) - obj.schemeAmount
13824 kshitij.so 168 
                    obj.maxNlc = obj.dp - (obj.dp * obj.minDiscount/100) - obj.schemeAmount
169 
                    obj.maxprice = obj.maxNlc
13755 kshitij.so 170 
            else:
171 
                """No rule found, use saholic min nlc as max and min R-Nlc"""
172 
                if identifier is not None:
173 
                    it = Item.query.filter_by(catalog_item_id=identifier).first()
174 
                    try:
175 
                        vendorPricing = inventory_client.getAllItemPricing(it.id)
176 
                        min_nlc = min(pricing.nlc for pricing in vendorPricing)
177 
                        obj.maxNlc = min_nlc
13824 kshitij.so 178 
                        obj.minNlc = min_nlc
179 
                        obj.maxprice = obj.maxNlc
13755 kshitij.so 180 
                    except:
181 
                        pass
182 
        populated = populated + 300
183 
        time.sleep(10)
184 
 
14306 kshitij.so 185 
def addManualDealsInfo():
186 
    for sku in LATEST_UPDATED_ITEMS:
187 
        manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':sku.source_id, 'sku':sku._id}))
188 
        if len(manualDeal) > 0:
189 
            sku.dealType = manualDeal[0]['dealType']
14310 kshitij.so 190 
 
191 
 
192 
    """Remove deal flag from expired deals"""
14311 kshitij.so 193 
    manualDeals = list(get_mongo_connection().Catalog.Deals.find({'dealType':1}))
14310 kshitij.so 194 
    for manualDeal in manualDeals:
14389 kshitij.so 195 
        d_manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':manualDeal['source_id'], 'sku':manualDeal['_id']}))
196 
        if len(d_manualDeal) > 0:
14310 kshitij.so 197 
            continue
198 
        else:
199 
            get_mongo_connection().Catalog.Deals.update({'_id':manualDeal['_id']},{"$set":{'dealType':0}},upsert=False, multi=True)
200 
 
14306 kshitij.so 201 
 
13755 kshitij.so 202 
def calculateNlcPoints():
203 
    global LATEST_UPDATED_ITEMS
204 
    for sku in LATEST_UPDATED_ITEMS:
205 
        if sku.maxNlc and sku.minNlc:
13824 kshitij.so 206 
 
207 
            """Create map - TODO"""
208 
 
209 
            if sku.status == 2:
210 
                eolWeight = .60
211 
            else:
212 
                eolWeight = 1.0
213 
            if sku.category_id == 3:
214 
                basePointPercentage = 5.0
215 
                maxNlcPoints = 200
216 
            elif sku.category_id == 5:
217 
                basePointPercentage = 8.0
218 
                maxNlcPoints = 150
219 
            else:
220 
                basePointPercentage = 10.0
221 
                maxNlcPoints = 150
13755 kshitij.so 222 
            discFromMinNlc = (sku.minNlc - sku.available_price)/sku.available_price *100
223 
            discFromMaxNlc = (sku.maxNlc - sku.available_price)/sku.available_price *100
224 
            if discFromMinNlc > 0:
13824 kshitij.so 225 
                nlcPoints = 100/basePointPercentage * discFromMinNlc
13755 kshitij.so 226 
            elif discFromMinNlc < 0 and discFromMaxNlc > 0:
227 
                nlcPoints = 0
228 
            else:
13824 kshitij.so 229 
                nlcPoints = 100/basePointPercentage * discFromMinNlc
230 
            if (min(nlcPoints,maxNlcPoints)) > 0:
231 
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints)) * eolWeight
232 
            else:
233 
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints))
13755 kshitij.so 234 
        else:
235 
            sku.nlcPoints = 0
236 
 
237 
def commitData():
238 
    global LATEST_UPDATED_ITEMS
239 
    for sku in LATEST_UPDATED_ITEMS:
240 
        #get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{'$set' : sku.__dict__},upsert=True,multi=True)
14020 amit.gupta 241 
        get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{"$set":sku.__dict__},upsert=True)
13755 kshitij.so 242 
 
243 
 
244 
def addBestSellerPoints():
245 
    allItems = list(get_mongo_connection().Catalog.Deals.find({}))
246 
    for sku in allItems:
13824 kshitij.so 247 
        bestSellerPoints = list(get_mongo_connection().Catalog.BestSellerPoints.find( {"$and":[{'min_rank': { "$lte": sku['rank'] } }, {'max_rank': { "$gte": sku['rank'] } } , { 'category_id' : sku['category_id'] }, { 'source_id' : sku['source_id'] }] } ))
13755 kshitij.so 248 
        if len(bestSellerPoints) > 0:
13824 kshitij.so 249 
            print bestSellerPoints[0]['points']
250 
            if (bestSellerPoints[0]['points']) > 0:
251 
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points']) * bestSellerPoints[0]['weightage']
252 
            else:
253 
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points'])
13755 kshitij.so 254 
        else:
13824 kshitij.so 255 
            sku['bestSellerPoints'] = -100
14114 kshitij.so 256 
        #sku['totalPoints'] = sku['bestSellerPoints'] + sku['nlcPoints']
257 
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'bestSellerPoints':sku['bestSellerPoints']}},multi=False)
258 
 
14390 kshitij.so 259 
    shortageSkus = get_mongo_connection().Catalog.MasterData.find({"$and":[{'is_shortage': 1 }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).distinct('_id')
260 
    print "Shortage skus"
261 
    print shortageSkus
14388 kshitij.so 262 
 
14114 kshitij.so 263 
    for sku in allItems:
264 
        deal_item = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':sku['skuBundleId']}).sort('bestSellerPoints',pymongo.DESCENDING).limit(1))
14142 kshitij.so 265 
        sku['catalogBestSellerPoints'] = deal_item[0]['bestSellerPoints']
14388 kshitij.so 266 
        shortagePoints = 50 if sku['_id'] in shortageSkus else 0
14391 kshitij.so 267 
        print "Shortage points for ",sku['_id']
268 
        print shortagePoints
14388 kshitij.so 269 
        sku['totalPoints'] = sku['catalogBestSellerPoints'] + sku['nlcPoints'] + shortagePoints
14114 kshitij.so 270 
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'catalogBestSellerPoints':sku['catalogBestSellerPoints'],'totalPoints':sku['totalPoints']}},multi=False)
13755 kshitij.so 271 
 
14325 kshitij.so 272 
 
273 
def populateNegativeDeals():
274 
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
275 
    mc.set("negative_deals", negativeDeals, 600)
276 
 
13828 kshitij.so 277 
def elimiateSimilarDeals():
13912 kshitij.so 278 
    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')
279 
    for skuBundleId in allItems:
280 
        print skuBundleId
281 
        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
13828 kshitij.so 282 
        bestPrice = float("inf")
283 
        bestOne = None
284 
        bestSellerPoints = 0
285 
        toUpdate = []
286 
        for similarItem in similarItems:
14331 kshitij.so 287 
            if mc.get("negative_deals") is None:
14325 kshitij.so 288 
                populateNegativeDeals()
14705 kshitij.so 289 
            try:
290 
                cashBack = getCashBack(similarItem['_id'], similarItem['source_id'], similarItem['category_id'], mc, options.mongoHost)
291 
                if not cashBack or cashBack.get('cash_back_status')!=1:
292 
                    pass
293 
                else:
294 
                    if cashBack['cash_back_type'] ==1:
295 
                        similarItem['available_price'] = similarItem['available_price'] - similarItem['available_price'] * float(cashBack['cash_back'])/100
296 
                    elif cashBack['cash_back_type'] ==2:
297 
                        similarItem['available_price'] = similarItem['available_price'] - float(cashBack['cash_back'])
298 
                    else:
299 
                        pass
300 
            except Exception as cashBackEx:
301 
                print cashBackEx
302 
                print "Error calculating cashback."
14331 kshitij.so 303 
            if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
13828 kshitij.so 304 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
305 
                continue
306 
            if similarItem['available_price'] < bestPrice:
307 
                bestOne = similarItem
308 
                bestPrice = similarItem['available_price']
309 
                bestSellerPoints = similarItem['bestSellerPoints']
310 
            elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
311 
                bestOne = similarItem
312 
                bestPrice = similarItem['available_price']
313 
                bestSellerPoints = similarItem['bestSellerPoints']
314 
            else:
315 
                pass
316 
        if bestOne is not None:
317 
            for similarItem in similarItems:
318 
                toUpdate.append(similarItem['_id'])
319 
            toUpdate.remove(bestOne['_id'])
320 
            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
321 
        if len(toUpdate) > 0:
13908 kshitij.so 322 
            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
13755 kshitij.so 323 
 
14741 kshitij.so 324 
def elimiateSimilarDealsOld():
325 
    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')
326 
    for skuBundleId in allItems:
327 
        print skuBundleId
328 
        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
329 
        bestPrice = float("inf")
330 
        bestOne = None
331 
        bestSellerPoints = 0
332 
        toUpdate = []
333 
        for similarItem in similarItems:
334 
            if mc.get("negative_deals") is None:
335 
                populateNegativeDeals()
336 
            if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
337 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDealOld':0 }})
338 
                continue
339 
            if similarItem['available_price'] < bestPrice:
340 
                bestOne = similarItem
341 
                bestPrice = similarItem['available_price']
342 
                bestSellerPoints = similarItem['bestSellerPoints']
343 
            elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
344 
                bestOne = similarItem
345 
                bestPrice = similarItem['available_price']
346 
                bestSellerPoints = similarItem['bestSellerPoints']
347 
            else:
348 
                pass
349 
        if bestOne is not None:
350 
            for similarItem in similarItems:
351 
                toUpdate.append(similarItem['_id'])
352 
            toUpdate.remove(bestOne['_id'])
353 
            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDealOld':1 }})
354 
        if len(toUpdate) > 0:
355 
            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDealOld':0 }},upsert=False, multi=True)
356 
 
357 
 
358 
 
13755 kshitij.so 359 
def main():
14263 kshitij.so 360 
    try:
361 
        populateStuff()
362 
        calculateNlc()
14306 kshitij.so 363 
        addManualDealsInfo()
14263 kshitij.so 364 
    finally:
365 
        session.close()
13755 kshitij.so 366 
    calculateNlcPoints()
367 
    commitData()
368 
    addBestSellerPoints()
13828 kshitij.so 369 
    elimiateSimilarDeals()
14741 kshitij.so 370 
    elimiateSimilarDealsOld()
13755 kshitij.so 371 
 
13828 kshitij.so 372 
 
13755 kshitij.so 373 
if __name__=='__main__':
374 
    main()