Subversion Repositories SmartDukaan

Rev

Rev 16171 | Rev 16405 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13755 kshitij.so 1
import pymongo
2
from elixir import *
3
from shop2020.model.v1.catalog.impl import DataService
4
from shop2020.model.v1.catalog.impl.DataService import Item
5
from shop2020.clients.InventoryClient import InventoryClient
6
from dtr.utils.utils import to_java_date
7
from datetime import datetime, timedelta
8
import time
14258 kshitij.so 9
import optparse
14325 kshitij.so 10
from dtr.storage.MemCache import MemCache
13755 kshitij.so 11
 
14258 kshitij.so 12
parser = optparse.OptionParser()
14263 kshitij.so 13
parser.add_option("-H", "--host", dest="hostname",
14258 kshitij.so 14
                      default="localhost",
15
                      type="string", help="The HOST where the DB server is running",
16
                      metavar="host")
17
parser.add_option("-m", "--m", dest="mongoHost",
18
                      default="localhost",
19
                      type="string", help="The HOST where the mongo server is running",
20
                      metavar="mongo_host")
13755 kshitij.so 21
 
14258 kshitij.so 22
(options, args) = parser.parse_args()
14325 kshitij.so 23
 
24
mc = MemCache(options.mongoHost)
25
 
14258 kshitij.so 26
DataService.initialize(db_hostname=options.hostname)
27
 
13755 kshitij.so 28
con = None
15907 kshitij.so 29
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
13755 kshitij.so 30
DISCOUNT_TYPE = {'MRP':1,'DP':2}
31
LATEST_UPDATED_ITEMS = []
16253 kshitij.so 32
STATUS_WEIGHTAGE = {1 : 1.0, 2 : 2.0, 3 : 1.0, 4 : 0.5}
13755 kshitij.so 33
 
34
now = datetime.now()
35
 
36
class __SkuInfo:
37
 
38
    def __init__(self, _id, skuBundleId, category_id, mrp, available_price, source_id, rank, maxNlc, minNlc, schemeAmount, minDiscount, \
16019 kshitij.so 39
                 maxDiscount, discountType, dp, nlcPoints, status, in_stock, maxprice, brand, dealType, dealPoints, brand_id, manualDealThresholdPrice,\
16253 kshitij.so 40
                 codAvailable,showDp):
13755 kshitij.so 41
        self._id = _id
42
        self.skuBundleId = skuBundleId
43
        self.category_id = category_id
44
        self.mrp = mrp
45
        self.available_price = available_price
46
        self.source_id = source_id
47
        self.rank = rank
48
        self.maxNlc = maxNlc
49
        self.minNlc = minNlc
50
        self.schemeAmount = schemeAmount
51
        self.minDiscount = minDiscount
52
        self.maxDiscount = maxDiscount
53
        self.discountType = discountType
54
        self.dp = dp
55
        self.nlcPoints = nlcPoints
13824 kshitij.so 56
        self.status = status
57
        self.in_stock = in_stock
58
        self.maxprice = maxprice
14035 kshitij.so 59
        self.brand = brand
14306 kshitij.so 60
        self.dealType = dealType
14388 kshitij.so 61
        self.dealPoints = dealPoints
15063 kshitij.so 62
        self.brand_id = brand_id
15271 kshitij.so 63
        self.manualDealThresholdPrice = manualDealThresholdPrice
16019 kshitij.so 64
        self.codAvailable = codAvailable
16253 kshitij.so 65
        self.showDp = showDp
13755 kshitij.so 66
 
67
 
14258 kshitij.so 68
def get_mongo_connection(host=options.mongoHost, port=27017):
13755 kshitij.so 69
    global con
70
    if con is None:
71
        print "Establishing connection %s host and port %d" %(host,port)
72
        try:
73
            con = pymongo.MongoClient(host, port)
74
        except Exception, e:
75
            print e
76
            return None
77
    return con
78
 
79
def populateStuff():
80
    print "Inside populate"
81
    global LATEST_UPDATED_ITEMS
82
    """Fetch latest updated items across portals
83
       and calculate max and min R-Nlc"""
84
    offset= 0
85
    while(True):
86
        print "Fetching records offset %d and limit %d" %(offset,300)
87
        topSkus = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'updatedOn': { "$gt": to_java_date(now - timedelta(hours=4))} }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).skip(offset).limit(300))
88
        if len((topSkus)) == 0:
89
            break
90
        #topSkus = collection.find( {'_id':664})
91
        for sku in topSkus:
13908 kshitij.so 92
            """Fix this """
93
            #TODO Compute deal flags else where.
13755 kshitij.so 94
            info = __SkuInfo(sku['_id'], sku['skuBundleId'], sku['category_id'], sku['mrp'], sku['available_price'], sku['source_id'], sku['rank'], None, None, 0.0, None, \
16253 kshitij.so 95
                             None, None, None, None, sku['status'], sku['in_stock'],sku['maxPrice'],sku['brand'].strip().upper(), 0, 0, sku['brand_id'], None, sku['codAvailable'], 0)
15854 kshitij.so 96
            exceptionalNlc = list(get_mongo_connection().Catalog.ExceptionalNlc.find( {"$and" : [ {'skuBundleId':info.skuBundleId}, {'overrideNlc':1} ]} ))
13755 kshitij.so 97
            if len(exceptionalNlc) > 0:
98
                """Exceptional nlc found, no need to calculate max and min R-nlc"""
99
                info.maxNlc = exceptionalNlc[0]['maxNlc']
100
                info.minNlc = exceptionalNlc[0]['minNlc']
14842 kshitij.so 101
                if info.maxprice == 0:
102
                    info.maxprice = exceptionalNlc[0]['maxNlc']
13755 kshitij.so 103
                LATEST_UPDATED_ITEMS.append(info)
104
                continue
105
 
15854 kshitij.so 106
            skuSchemeDetails = list(get_mongo_connection().Catalog.SkuSchemeDetails.find( {'skuBundleId':info.skuBundleId}))
13755 kshitij.so 107
            if len(skuSchemeDetails) > 0:
108
                """Sku scheme details, populate scheme amount (Recently added)"""
13824 kshitij.so 109
 
110
                #TODO Add start date and end date of scehems
111
 
13755 kshitij.so 112
                info.schemeAmount = float(skuSchemeDetails[0]['schemeAmount'])
113
 
15854 kshitij.so 114
            skuDealerPrices = list(get_mongo_connection().Catalog.SkuDealerPrices.find( {'skuBundleId':info.skuBundleId} ) )
13755 kshitij.so 115
            if len(skuDealerPrices) > 0:
116
                info.dp = skuDealerPrices[0]['dp']
16253 kshitij.so 117
                info.showDp = skuDealerPrices[0]['showDp']
15854 kshitij.so 118
            skuDiscount = list(get_mongo_connection().Catalog.SkuDiscountInfo.find( {'skuBundleId':info.skuBundleId} ) )
13755 kshitij.so 119
            if len(skuDiscount) > 0:
120
                """Sku rule found, populate max , min Discount and discount type"""
121
                info.maxDiscount = skuDiscount[0]['max_discount']
122
                info.minDiscount = skuDiscount[0]['min_discount']
123
                info.discountType = DISCOUNT_TYPE.get(skuDiscount[0]['discountType'].upper())
124
                LATEST_UPDATED_ITEMS.append(info)
125
                continue
126
 
14035 kshitij.so 127
            categoryDiscount = list(get_mongo_connection().Catalog.CategoryDiscount.find( {"$and" : [{'brand':sku['brand'].strip().upper()}, {'category_id':sku['category_id']} ]} ))
13755 kshitij.so 128
            if len(categoryDiscount) > 0:
129
                info.maxDiscount = categoryDiscount[0]['max_discount']
130
                info.minDiscount = categoryDiscount[0]['min_discount']
131
                info.discountType = DISCOUNT_TYPE.get(categoryDiscount[0]['discountType'].upper())
132
 
133
            LATEST_UPDATED_ITEMS.append(info)
134
        offset = offset + 300
135
    for lol in LATEST_UPDATED_ITEMS:
136
        print lol.__dict__
137
 
138
 
139
def calculateNlc():
140
    global LATEST_UPDATED_ITEMS
141
    populated = 0
142
    while(populated <= len(LATEST_UPDATED_ITEMS)):
143
        inventory_client = InventoryClient().get_client()
144
        for obj in LATEST_UPDATED_ITEMS[populated:300+populated]:
145
            if obj.maxNlc > 0 and obj.minNlc > 0:
146
                continue
147
            saholic_sku = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'skuBundleId': obj.skuBundleId}, { 'source_id' : SOURCE_MAP.get('SAHOLIC')}] }))
148
            identifier = None
149
            if len(saholic_sku) > 0:
150
                identifier = saholic_sku[0]['identifier']
151
            if obj.discountType == DISCOUNT_TYPE.get('MRP'):
152
                if obj.mrp == 0:
153
                    """Now mrp is zero, so we have to use saholic MRP"""
154
                    if identifier is not None:
155
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
156
                        obj.mrp = it.mrp
157
                if obj.mrp > 0:
14842 kshitij.so 158
                    print obj._id
13755 kshitij.so 159
                    obj.minNlc = obj.mrp - (obj.mrp * obj.maxDiscount/100) - obj.schemeAmount
13824 kshitij.so 160
                    obj.maxNlc = obj.mrp - (obj.mrp * obj.minDiscount/100) - obj.schemeAmount
14842 kshitij.so 161
                    if obj.maxprice == 0:
162
                        obj.maxprice = obj.maxNlc  
13755 kshitij.so 163
            elif obj.discountType == DISCOUNT_TYPE.get('DP'):
164
                if obj.dp == 0:
165
                    """Now dp is zero, so we have to use saholic minimum dp for item"""
166
                    if identifier is not None:
167
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
168
                        try:
169
                            vendorPricing = inventory_client.getAllItemPricing(it.id)
170
                            min_dp = min(pricing.dealerPrice for pricing in vendorPricing)
171
                            obj.dp = min_dp 
16253 kshitij.so 172
                            obj.showDp = 1
13755 kshitij.so 173
                        except:
174
                            pass
175
                if obj.dp > 0:
176
                    obj.minNlc = obj.dp - (obj.dp * obj.maxDiscount/100) - obj.schemeAmount
13824 kshitij.so 177
                    obj.maxNlc = obj.dp - (obj.dp * obj.minDiscount/100) - obj.schemeAmount
14842 kshitij.so 178
                    if obj.maxprice == 0:
179
                        obj.maxprice = obj.maxNlc  
13755 kshitij.so 180
            else:
181
                """No rule found, use saholic min nlc as max and min R-Nlc"""
182
                if identifier is not None:
183
                    it = Item.query.filter_by(catalog_item_id=identifier).first()
184
                    try:
185
                        vendorPricing = inventory_client.getAllItemPricing(it.id)
186
                        min_nlc = min(pricing.nlc for pricing in vendorPricing)
187
                        obj.maxNlc = min_nlc
13824 kshitij.so 188
                        obj.minNlc = min_nlc
14842 kshitij.so 189
                        if obj.maxprice == 0:
190
                            obj.maxprice = obj.maxNlc   
13755 kshitij.so 191
                    except:
192
                        pass
193
        populated = populated + 300
194
        time.sleep(10) 
195
 
14306 kshitij.so 196
def addManualDealsInfo():
197
    for sku in LATEST_UPDATED_ITEMS:
198
        manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':sku.source_id, 'sku':sku._id}))
199
        if len(manualDeal) > 0:
200
            sku.dealType = manualDeal[0]['dealType']
15271 kshitij.so 201
            if sku.available_price <= manualDeal[0]['dealThresholdPrice']:
202
                sku.dealPoints = manualDeal[0]['dealPoints']
203
                sku.manualDealThresholdPrice = manualDeal[0]['dealThresholdPrice']
204
            else:
205
                sku.dealPoints = 0
14310 kshitij.so 206
 
207
    """Remove deal flag from expired deals"""
14311 kshitij.so 208
    manualDeals = list(get_mongo_connection().Catalog.Deals.find({'dealType':1}))
14310 kshitij.so 209
    for manualDeal in manualDeals:
14389 kshitij.so 210
        d_manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':manualDeal['source_id'], 'sku':manualDeal['_id']}))
211
        if len(d_manualDeal) > 0:
14310 kshitij.so 212
            continue
213
        else:
15271 kshitij.so 214
            get_mongo_connection().Catalog.Deals.update({'_id':manualDeal['_id']},{"$set":{'dealType':0,'dealPoints':0,'manualDealThresholdPrice':None}},upsert=False, multi=False)
14310 kshitij.so 215
 
14306 kshitij.so 216
 
13755 kshitij.so 217
def calculateNlcPoints():
218
    global LATEST_UPDATED_ITEMS
14842 kshitij.so 219
    print "inside nlc oints"
13755 kshitij.so 220
    for sku in LATEST_UPDATED_ITEMS:
221
        if sku.maxNlc and sku.minNlc:
14842 kshitij.so 222
            print sku._id
13824 kshitij.so 223
 
224
            """Create map - TODO"""
225
 
226
            if sku.status == 2:
227
                eolWeight = .60
228
            else:
229
                eolWeight = 1.0
230
            if sku.category_id == 3:
231
                basePointPercentage = 5.0
232
                maxNlcPoints = 200
233
            elif sku.category_id == 5:
234
                basePointPercentage = 8.0
235
                maxNlcPoints = 150
236
            else:
237
                basePointPercentage = 10.0
238
                maxNlcPoints = 150
14842 kshitij.so 239
            discFromMinNlc = float((sku.minNlc - sku.available_price))/sku.available_price *100
240
            discFromMaxNlc = float((sku.maxNlc - sku.available_price))/sku.available_price *100
241
            print discFromMinNlc
242
            print discFromMaxNlc
13755 kshitij.so 243
            if discFromMinNlc > 0:
13824 kshitij.so 244
                nlcPoints = 100/basePointPercentage * discFromMinNlc
13755 kshitij.so 245
            elif discFromMinNlc < 0 and discFromMaxNlc > 0:
246
                nlcPoints = 0
247
            else:
13824 kshitij.so 248
                nlcPoints = 100/basePointPercentage * discFromMinNlc
249
            if (min(nlcPoints,maxNlcPoints)) > 0:
250
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints)) * eolWeight
251
            else:
252
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints))
13755 kshitij.so 253
        else:
254
            sku.nlcPoints = 0
255
 
256
def commitData():
257
    global LATEST_UPDATED_ITEMS
258
    for sku in LATEST_UPDATED_ITEMS:
259
        #get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{'$set' : sku.__dict__},upsert=True,multi=True)
14020 amit.gupta 260
        get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{"$set":sku.__dict__},upsert=True)
13755 kshitij.so 261
 
262
 
263
def addBestSellerPoints():
264
    allItems = list(get_mongo_connection().Catalog.Deals.find({})) 
265
    for sku in allItems:
13824 kshitij.so 266
        bestSellerPoints = list(get_mongo_connection().Catalog.BestSellerPoints.find( {"$and":[{'min_rank': { "$lte": sku['rank'] } }, {'max_rank': { "$gte": sku['rank'] } } , { 'category_id' : sku['category_id'] }, { 'source_id' : sku['source_id'] }] } ))
13755 kshitij.so 267
        if len(bestSellerPoints) > 0:
13824 kshitij.so 268
            print bestSellerPoints[0]['points']
269
            if (bestSellerPoints[0]['points']) > 0:
16253 kshitij.so 270
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points']) * bestSellerPoints[0]['weightage'] * STATUS_WEIGHTAGE.get(sku['status'])
13824 kshitij.so 271
            else:
272
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points'])
13755 kshitij.so 273
        else:
16253 kshitij.so 274
            sku['bestSellerPoints'] = -120
14114 kshitij.so 275
        #sku['totalPoints'] = sku['bestSellerPoints'] + sku['nlcPoints']
276
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'bestSellerPoints':sku['bestSellerPoints']}},multi=False)
277
 
14390 kshitij.so 278
    shortageSkus = get_mongo_connection().Catalog.MasterData.find({"$and":[{'is_shortage': 1 }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).distinct('_id')
279
    print "Shortage skus"
280
    print shortageSkus
14388 kshitij.so 281
 
14114 kshitij.so 282
    for sku in allItems:
283
        deal_item = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':sku['skuBundleId']}).sort('bestSellerPoints',pymongo.DESCENDING).limit(1))
14142 kshitij.so 284
        sku['catalogBestSellerPoints'] = deal_item[0]['bestSellerPoints']
14388 kshitij.so 285
        shortagePoints = 50 if sku['_id'] in shortageSkus else 0
14391 kshitij.so 286
        print "Shortage points for ",sku['_id']
287
        print shortagePoints
15271 kshitij.so 288
        sku['totalPoints'] = sku['catalogBestSellerPoints'] + sku['nlcPoints'] + shortagePoints + sku['dealPoints'] 
14114 kshitij.so 289
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'catalogBestSellerPoints':sku['catalogBestSellerPoints'],'totalPoints':sku['totalPoints']}},multi=False)
15271 kshitij.so 290
 
291
 
13755 kshitij.so 292
 
14325 kshitij.so 293
def populateNegativeDeals():
294
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
295
    mc.set("negative_deals", negativeDeals, 600)   
296
 
13828 kshitij.so 297
def elimiateSimilarDeals():
13912 kshitij.so 298
    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')
299
    for skuBundleId in allItems:
300
        print skuBundleId
301
        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
13828 kshitij.so 302
        bestPrice = float("inf")
303
        bestOne = None
304
        bestSellerPoints = 0
305
        toUpdate = []
16019 kshitij.so 306
        prepaidBestPrice = float("inf")
307
        prepaidBestOne = None
308
        prepaidBestSellerPoints = 0
13828 kshitij.so 309
        for similarItem in similarItems:
16019 kshitij.so 310
            if similarItem['codAvailable'] ==1:
311
                if mc.get("negative_deals") is None:
312
                    populateNegativeDeals()
16171 kshitij.so 313
                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
16019 kshitij.so 314
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
315
                    continue
316
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
317
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
318
                    continue
319
                if similarItem['available_price'] < bestPrice:
320
                    bestOne = similarItem
321
                    bestPrice = similarItem['available_price']
322
                    bestSellerPoints = similarItem['bestSellerPoints']
323
                elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
324
                    bestOne = similarItem
325
                    bestPrice = similarItem['available_price']
326
                    bestSellerPoints = similarItem['bestSellerPoints']
327
                else:
328
                    pass
13828 kshitij.so 329
            else:
16019 kshitij.so 330
                if mc.get("negative_deals") is None:
331
                    populateNegativeDeals()
16171 kshitij.so 332
                if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
16019 kshitij.so 333
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
334
                    continue
335
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
336
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
337
                    continue
338
                if similarItem['available_price'] < prepaidBestPrice:
339
                    prepaidBestOne = similarItem
340
                    prepaidBestPrice = similarItem['available_price']
341
                    prepaidBestSellerPoints = similarItem['bestSellerPoints']
342
                elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
343
                    prepaidBestOne = similarItem
344
                    prepaidBestPrice = similarItem['available_price']
345
                    prepaidBestSellerPoints = similarItem['bestSellerPoints']
346
                else:
347
                    pass
16026 kshitij.so 348
        if bestOne is not None or prepaidBestOne is not None:
13828 kshitij.so 349
            for similarItem in similarItems:
350
                toUpdate.append(similarItem['_id'])
16026 kshitij.so 351
            if bestOne is not None:
352
                toUpdate.remove(bestOne['_id'])
353
                get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
354
            if prepaidBestOne is not None:
16071 kshitij.so 355
                if bestOne is not None:
356
                    if prepaidBestOne['available_price'] < bestOne['available_price']: 
357
                        toUpdate.remove(prepaidBestOne['_id'])
358
                        get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
359
                else:
360
                    toUpdate.remove(prepaidBestOne['_id'])
361
                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
13828 kshitij.so 362
        if len(toUpdate) > 0:
16019 kshitij.so 363
            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
13755 kshitij.so 364
 
365
def main():
14263 kshitij.so 366
    try:
367
        populateStuff()
368
        calculateNlc()
14306 kshitij.so 369
        addManualDealsInfo()
14263 kshitij.so 370
    finally:
371
        session.close()
13755 kshitij.so 372
    calculateNlcPoints()
373
    commitData()
374
    addBestSellerPoints()
13828 kshitij.so 375
    elimiateSimilarDeals()
13755 kshitij.so 376
 
13828 kshitij.so 377
 
13755 kshitij.so 378
if __name__=='__main__':
14842 kshitij.so 379
    main()