Subversion Repositories SmartDukaan

Rev

Rev 16507 | Rev 16635 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13755 kshitij.so 1
import pymongo
2
from elixir import *
3
from shop2020.model.v1.catalog.impl import DataService
4
from shop2020.model.v1.catalog.impl.DataService import Item
5
from shop2020.clients.InventoryClient import InventoryClient
6
from dtr.utils.utils import to_java_date
7
from datetime import datetime, timedelta
8
import time
14258 kshitij.so 9
import optparse
14325 kshitij.so 10
from dtr.storage.MemCache import MemCache
13755 kshitij.so 11
 
14258 kshitij.so 12
parser = optparse.OptionParser()
14263 kshitij.so 13
parser.add_option("-H", "--host", dest="hostname",
14258 kshitij.so 14
                      default="localhost",
15
                      type="string", help="The HOST where the DB server is running",
16
                      metavar="host")
17
parser.add_option("-m", "--m", dest="mongoHost",
18
                      default="localhost",
19
                      type="string", help="The HOST where the mongo server is running",
20
                      metavar="mongo_host")
13755 kshitij.so 21
 
14258 kshitij.so 22
(options, args) = parser.parse_args()
14325 kshitij.so 23
 
24
mc = MemCache(options.mongoHost)
25
 
14258 kshitij.so 26
DataService.initialize(db_hostname=options.hostname)
27
 
13755 kshitij.so 28
con = None
16405 kshitij.so 29
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6}
13755 kshitij.so 30
DISCOUNT_TYPE = {'MRP':1,'DP':2}
31
LATEST_UPDATED_ITEMS = []
16253 kshitij.so 32
STATUS_WEIGHTAGE = {1 : 1.0, 2 : 2.0, 3 : 1.0, 4 : 0.5}
13755 kshitij.so 33
 
34
now = datetime.now()
35
 
36
class __SkuInfo:
37
 
38
    def __init__(self, _id, skuBundleId, category_id, mrp, available_price, source_id, rank, maxNlc, minNlc, schemeAmount, minDiscount, \
16534 kshitij.so 39
                 maxDiscount, discountType, dp, nlcPoints, status, in_stock, maxprice, brand, dealType, brand_id, manualDealThresholdPrice,\
16456 kshitij.so 40
                 codAvailable,showDp,gross_price):
13755 kshitij.so 41
        self._id = _id
42
        self.skuBundleId = skuBundleId
43
        self.category_id = category_id
44
        self.mrp = mrp
45
        self.available_price = available_price
46
        self.source_id = source_id
47
        self.rank = rank
48
        self.maxNlc = maxNlc
49
        self.minNlc = minNlc
50
        self.schemeAmount = schemeAmount
51
        self.minDiscount = minDiscount
52
        self.maxDiscount = maxDiscount
53
        self.discountType = discountType
54
        self.dp = dp
55
        self.nlcPoints = nlcPoints
13824 kshitij.so 56
        self.status = status
57
        self.in_stock = in_stock
58
        self.maxprice = maxprice
14035 kshitij.so 59
        self.brand = brand
14306 kshitij.so 60
        self.dealType = dealType
15063 kshitij.so 61
        self.brand_id = brand_id
15271 kshitij.so 62
        self.manualDealThresholdPrice = manualDealThresholdPrice
16019 kshitij.so 63
        self.codAvailable = codAvailable
16253 kshitij.so 64
        self.showDp = showDp
16456 kshitij.so 65
        self.gross_price = gross_price
13755 kshitij.so 66
 
67
 
14258 kshitij.so 68
def get_mongo_connection(host=options.mongoHost, port=27017):
13755 kshitij.so 69
    global con
70
    if con is None:
71
        print "Establishing connection %s host and port %d" %(host,port)
72
        try:
73
            con = pymongo.MongoClient(host, port)
74
        except Exception, e:
75
            print e
76
            return None
77
    return con
78
 
79
def populateStuff():
80
    print "Inside populate"
81
    global LATEST_UPDATED_ITEMS
82
    """Fetch latest updated items across portals
83
       and calculate max and min R-Nlc"""
84
    offset= 0
85
    while(True):
86
        print "Fetching records offset %d and limit %d" %(offset,300)
87
        topSkus = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'updatedOn': { "$gt": to_java_date(now - timedelta(hours=4))} }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).skip(offset).limit(300))
88
        if len((topSkus)) == 0:
89
            break
90
        #topSkus = collection.find( {'_id':664})
91
        for sku in topSkus:
13908 kshitij.so 92
            """Fix this """
93
            #TODO Compute deal flags else where.
13755 kshitij.so 94
            info = __SkuInfo(sku['_id'], sku['skuBundleId'], sku['category_id'], sku['mrp'], sku['available_price'], sku['source_id'], sku['rank'], None, None, 0.0, None, \
16534 kshitij.so 95
                             None, None, None, None, sku['status'], sku['in_stock'],sku['maxPrice'],sku['brand'].strip().upper(), 0, sku['brand_id'], None, sku['codAvailable'], 0, sku['gross_price'])
15854 kshitij.so 96
            exceptionalNlc = list(get_mongo_connection().Catalog.ExceptionalNlc.find( {"$and" : [ {'skuBundleId':info.skuBundleId}, {'overrideNlc':1} ]} ))
13755 kshitij.so 97
            if len(exceptionalNlc) > 0:
98
                """Exceptional nlc found, no need to calculate max and min R-nlc"""
99
                info.maxNlc = exceptionalNlc[0]['maxNlc']
100
                info.minNlc = exceptionalNlc[0]['minNlc']
14842 kshitij.so 101
                if info.maxprice == 0:
102
                    info.maxprice = exceptionalNlc[0]['maxNlc']
13755 kshitij.so 103
                LATEST_UPDATED_ITEMS.append(info)
104
                continue
105
 
15854 kshitij.so 106
            skuSchemeDetails = list(get_mongo_connection().Catalog.SkuSchemeDetails.find( {'skuBundleId':info.skuBundleId}))
13755 kshitij.so 107
            if len(skuSchemeDetails) > 0:
108
                """Sku scheme details, populate scheme amount (Recently added)"""
13824 kshitij.so 109
 
110
                #TODO Add start date and end date of scehems
111
 
13755 kshitij.so 112
                info.schemeAmount = float(skuSchemeDetails[0]['schemeAmount'])
113
 
15854 kshitij.so 114
            skuDealerPrices = list(get_mongo_connection().Catalog.SkuDealerPrices.find( {'skuBundleId':info.skuBundleId} ) )
13755 kshitij.so 115
            if len(skuDealerPrices) > 0:
116
                info.dp = skuDealerPrices[0]['dp']
16253 kshitij.so 117
                info.showDp = skuDealerPrices[0]['showDp']
15854 kshitij.so 118
            skuDiscount = list(get_mongo_connection().Catalog.SkuDiscountInfo.find( {'skuBundleId':info.skuBundleId} ) )
13755 kshitij.so 119
            if len(skuDiscount) > 0:
120
                """Sku rule found, populate max , min Discount and discount type"""
121
                info.maxDiscount = skuDiscount[0]['max_discount']
122
                info.minDiscount = skuDiscount[0]['min_discount']
123
                info.discountType = DISCOUNT_TYPE.get(skuDiscount[0]['discountType'].upper())
124
                LATEST_UPDATED_ITEMS.append(info)
125
                continue
126
 
14035 kshitij.so 127
            categoryDiscount = list(get_mongo_connection().Catalog.CategoryDiscount.find( {"$and" : [{'brand':sku['brand'].strip().upper()}, {'category_id':sku['category_id']} ]} ))
13755 kshitij.so 128
            if len(categoryDiscount) > 0:
129
                info.maxDiscount = categoryDiscount[0]['max_discount']
130
                info.minDiscount = categoryDiscount[0]['min_discount']
131
                info.discountType = DISCOUNT_TYPE.get(categoryDiscount[0]['discountType'].upper())
132
 
133
            LATEST_UPDATED_ITEMS.append(info)
134
        offset = offset + 300
135
    for lol in LATEST_UPDATED_ITEMS:
136
        print lol.__dict__
137
 
138
 
139
def calculateNlc():
140
    global LATEST_UPDATED_ITEMS
141
    populated = 0
142
    while(populated <= len(LATEST_UPDATED_ITEMS)):
143
        inventory_client = InventoryClient().get_client()
144
        for obj in LATEST_UPDATED_ITEMS[populated:300+populated]:
145
            if obj.maxNlc > 0 and obj.minNlc > 0:
146
                continue
147
            saholic_sku = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'skuBundleId': obj.skuBundleId}, { 'source_id' : SOURCE_MAP.get('SAHOLIC')}] }))
148
            identifier = None
149
            if len(saholic_sku) > 0:
150
                identifier = saholic_sku[0]['identifier']
151
            if obj.discountType == DISCOUNT_TYPE.get('MRP'):
152
                if obj.mrp == 0:
153
                    """Now mrp is zero, so we have to use saholic MRP"""
154
                    if identifier is not None:
155
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
156
                        obj.mrp = it.mrp
157
                if obj.mrp > 0:
14842 kshitij.so 158
                    print obj._id
13755 kshitij.so 159
                    obj.minNlc = obj.mrp - (obj.mrp * obj.maxDiscount/100) - obj.schemeAmount
13824 kshitij.so 160
                    obj.maxNlc = obj.mrp - (obj.mrp * obj.minDiscount/100) - obj.schemeAmount
14842 kshitij.so 161
                    if obj.maxprice == 0:
162
                        obj.maxprice = obj.maxNlc  
13755 kshitij.so 163
            elif obj.discountType == DISCOUNT_TYPE.get('DP'):
164
                if obj.dp == 0:
165
                    """Now dp is zero, so we have to use saholic minimum dp for item"""
166
                    if identifier is not None:
167
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
168
                        try:
169
                            vendorPricing = inventory_client.getAllItemPricing(it.id)
170
                            min_dp = min(pricing.dealerPrice for pricing in vendorPricing)
171
                            obj.dp = min_dp 
16253 kshitij.so 172
                            obj.showDp = 1
13755 kshitij.so 173
                        except:
174
                            pass
175
                if obj.dp > 0:
176
                    obj.minNlc = obj.dp - (obj.dp * obj.maxDiscount/100) - obj.schemeAmount
13824 kshitij.so 177
                    obj.maxNlc = obj.dp - (obj.dp * obj.minDiscount/100) - obj.schemeAmount
14842 kshitij.so 178
                    if obj.maxprice == 0:
179
                        obj.maxprice = obj.maxNlc  
13755 kshitij.so 180
            else:
181
                """No rule found, use saholic min nlc as max and min R-Nlc"""
182
                if identifier is not None:
183
                    it = Item.query.filter_by(catalog_item_id=identifier).first()
184
                    try:
185
                        vendorPricing = inventory_client.getAllItemPricing(it.id)
186
                        min_nlc = min(pricing.nlc for pricing in vendorPricing)
187
                        obj.maxNlc = min_nlc
13824 kshitij.so 188
                        obj.minNlc = min_nlc
14842 kshitij.so 189
                        if obj.maxprice == 0:
190
                            obj.maxprice = obj.maxNlc   
13755 kshitij.so 191
                    except:
192
                        pass
193
        populated = populated + 300
194
        time.sleep(10) 
195
 
14306 kshitij.so 196
def addManualDealsInfo():
197
    for sku in LATEST_UPDATED_ITEMS:
198
        manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':sku.source_id, 'sku':sku._id}))
199
        if len(manualDeal) > 0:
200
            sku.dealType = manualDeal[0]['dealType']
14310 kshitij.so 201
 
202
    """Remove deal flag from expired deals"""
14311 kshitij.so 203
    manualDeals = list(get_mongo_connection().Catalog.Deals.find({'dealType':1}))
14310 kshitij.so 204
    for manualDeal in manualDeals:
14389 kshitij.so 205
        d_manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':manualDeal['source_id'], 'sku':manualDeal['_id']}))
206
        if len(d_manualDeal) > 0:
14310 kshitij.so 207
            continue
208
        else:
16507 kshitij.so 209
            get_mongo_connection().Catalog.Deals.update({'_id':manualDeal['_id']},{"$set":{'dealType':0}},upsert=False, multi=False)
14306 kshitij.so 210
 
13755 kshitij.so 211
def calculateNlcPoints():
212
    global LATEST_UPDATED_ITEMS
14842 kshitij.so 213
    print "inside nlc oints"
13755 kshitij.so 214
    for sku in LATEST_UPDATED_ITEMS:
215
        if sku.maxNlc and sku.minNlc:
14842 kshitij.so 216
            print sku._id
13824 kshitij.so 217
 
218
            """Create map - TODO"""
219
 
220
            if sku.status == 2:
221
                eolWeight = .60
222
            else:
223
                eolWeight = 1.0
224
            if sku.category_id == 3:
225
                basePointPercentage = 5.0
226
                maxNlcPoints = 200
227
            elif sku.category_id == 5:
228
                basePointPercentage = 8.0
229
                maxNlcPoints = 150
230
            else:
231
                basePointPercentage = 10.0
232
                maxNlcPoints = 150
14842 kshitij.so 233
            discFromMinNlc = float((sku.minNlc - sku.available_price))/sku.available_price *100
234
            discFromMaxNlc = float((sku.maxNlc - sku.available_price))/sku.available_price *100
235
            print discFromMinNlc
236
            print discFromMaxNlc
13755 kshitij.so 237
            if discFromMinNlc > 0:
13824 kshitij.so 238
                nlcPoints = 100/basePointPercentage * discFromMinNlc
13755 kshitij.so 239
            elif discFromMinNlc < 0 and discFromMaxNlc > 0:
240
                nlcPoints = 0
241
            else:
13824 kshitij.so 242
                nlcPoints = 100/basePointPercentage * discFromMinNlc
243
            if (min(nlcPoints,maxNlcPoints)) > 0:
244
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints)) * eolWeight
245
            else:
246
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints))
13755 kshitij.so 247
        else:
248
            sku.nlcPoints = 0
249
 
250
def commitData():
251
    global LATEST_UPDATED_ITEMS
252
    for sku in LATEST_UPDATED_ITEMS:
253
        #get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{'$set' : sku.__dict__},upsert=True,multi=True)
14020 amit.gupta 254
        get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{"$set":sku.__dict__},upsert=True)
13755 kshitij.so 255
 
256
 
257
def addBestSellerPoints():
258
    allItems = list(get_mongo_connection().Catalog.Deals.find({})) 
259
    for sku in allItems:
13824 kshitij.so 260
        bestSellerPoints = list(get_mongo_connection().Catalog.BestSellerPoints.find( {"$and":[{'min_rank': { "$lte": sku['rank'] } }, {'max_rank': { "$gte": sku['rank'] } } , { 'category_id' : sku['category_id'] }, { 'source_id' : sku['source_id'] }] } ))
13755 kshitij.so 261
        if len(bestSellerPoints) > 0:
13824 kshitij.so 262
            print bestSellerPoints[0]['points']
263
            if (bestSellerPoints[0]['points']) > 0:
16253 kshitij.so 264
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points']) * bestSellerPoints[0]['weightage'] * STATUS_WEIGHTAGE.get(sku['status'])
13824 kshitij.so 265
            else:
266
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points'])
13755 kshitij.so 267
        else:
16253 kshitij.so 268
            sku['bestSellerPoints'] = -120
14114 kshitij.so 269
        #sku['totalPoints'] = sku['bestSellerPoints'] + sku['nlcPoints']
270
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'bestSellerPoints':sku['bestSellerPoints']}},multi=False)
271
 
14390 kshitij.so 272
    shortageSkus = get_mongo_connection().Catalog.MasterData.find({"$and":[{'is_shortage': 1 }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).distinct('_id')
273
    print "Shortage skus"
274
    print shortageSkus
14388 kshitij.so 275
 
14114 kshitij.so 276
    for sku in allItems:
277
        deal_item = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':sku['skuBundleId']}).sort('bestSellerPoints',pymongo.DESCENDING).limit(1))
14142 kshitij.so 278
        sku['catalogBestSellerPoints'] = deal_item[0]['bestSellerPoints']
14388 kshitij.so 279
        shortagePoints = 50 if sku['_id'] in shortageSkus else 0
14391 kshitij.so 280
        print "Shortage points for ",sku['_id']
281
        print shortagePoints
15271 kshitij.so 282
        sku['totalPoints'] = sku['catalogBestSellerPoints'] + sku['nlcPoints'] + shortagePoints + sku['dealPoints'] 
14114 kshitij.so 283
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'catalogBestSellerPoints':sku['catalogBestSellerPoints'],'totalPoints':sku['totalPoints']}},multi=False)
15271 kshitij.so 284
 
285
 
13755 kshitij.so 286
 
14325 kshitij.so 287
def populateNegativeDeals():
288
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
289
    mc.set("negative_deals", negativeDeals, 600)   
290
 
13828 kshitij.so 291
def elimiateSimilarDeals():
13912 kshitij.so 292
    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')
293
    for skuBundleId in allItems:
294
        print skuBundleId
295
        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
13828 kshitij.so 296
        bestPrice = float("inf")
297
        bestOne = None
298
        bestSellerPoints = 0
299
        toUpdate = []
16019 kshitij.so 300
        prepaidBestPrice = float("inf")
301
        prepaidBestOne = None
302
        prepaidBestSellerPoints = 0
13828 kshitij.so 303
        for similarItem in similarItems:
16019 kshitij.so 304
            if similarItem['codAvailable'] ==1:
305
                if mc.get("negative_deals") is None:
306
                    populateNegativeDeals()
16171 kshitij.so 307
                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
16019 kshitij.so 308
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
309
                    continue
310
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
311
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
312
                    continue
313
                if similarItem['available_price'] < bestPrice:
314
                    bestOne = similarItem
315
                    bestPrice = similarItem['available_price']
316
                    bestSellerPoints = similarItem['bestSellerPoints']
317
                elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
318
                    bestOne = similarItem
319
                    bestPrice = similarItem['available_price']
320
                    bestSellerPoints = similarItem['bestSellerPoints']
321
                else:
322
                    pass
13828 kshitij.so 323
            else:
16019 kshitij.so 324
                if mc.get("negative_deals") is None:
325
                    populateNegativeDeals()
16171 kshitij.so 326
                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
16019 kshitij.so 327
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
328
                    continue
329
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
330
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
331
                    continue
332
                if similarItem['available_price'] < prepaidBestPrice:
333
                    prepaidBestOne = similarItem
334
                    prepaidBestPrice = similarItem['available_price']
335
                    prepaidBestSellerPoints = similarItem['bestSellerPoints']
336
                elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
337
                    prepaidBestOne = similarItem
338
                    prepaidBestPrice = similarItem['available_price']
339
                    prepaidBestSellerPoints = similarItem['bestSellerPoints']
340
                else:
341
                    pass
16026 kshitij.so 342
        if bestOne is not None or prepaidBestOne is not None:
13828 kshitij.so 343
            for similarItem in similarItems:
344
                toUpdate.append(similarItem['_id'])
16026 kshitij.so 345
            if bestOne is not None:
346
                toUpdate.remove(bestOne['_id'])
347
                get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
348
            if prepaidBestOne is not None:
16071 kshitij.so 349
                if bestOne is not None:
350
                    if prepaidBestOne['available_price'] < bestOne['available_price']: 
351
                        toUpdate.remove(prepaidBestOne['_id'])
352
                        get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
353
                else:
354
                    toUpdate.remove(prepaidBestOne['_id'])
355
                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
13828 kshitij.so 356
        if len(toUpdate) > 0:
16019 kshitij.so 357
            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
13755 kshitij.so 358
 
359
def main():
14263 kshitij.so 360
    try:
361
        populateStuff()
362
        calculateNlc()
14306 kshitij.so 363
        addManualDealsInfo()
14263 kshitij.so 364
    finally:
365
        session.close()
13755 kshitij.so 366
    calculateNlcPoints()
367
    commitData()
368
    addBestSellerPoints()
13828 kshitij.so 369
    elimiateSimilarDeals()
13755 kshitij.so 370
 
13828 kshitij.so 371
 
13755 kshitij.so 372
if __name__=='__main__':
14842 kshitij.so 373
    main()