Subversion Repositories SmartDukaan

Rev

Rev 16019 | Rev 16071 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13755 kshitij.so 1 
import pymongo
2 
from elixir import *
3 
from shop2020.model.v1.catalog.impl import DataService
4 
from shop2020.model.v1.catalog.impl.DataService import Item
5 
from shop2020.clients.InventoryClient import InventoryClient
6 
from dtr.utils.utils import to_java_date
7 
from datetime import datetime, timedelta
8 
import time
14258 kshitij.so 9 
import optparse
14325 kshitij.so 10 
from dtr.storage.MemCache import MemCache
13755 kshitij.so 11 
 
14258 kshitij.so 12 
parser = optparse.OptionParser()
14263 kshitij.so 13 
parser.add_option("-H", "--host", dest="hostname",
14258 kshitij.so 14 
                      default="localhost",
15 
                      type="string", help="The HOST where the DB server is running",
16 
                      metavar="host")
17 
parser.add_option("-m", "--m", dest="mongoHost",
18 
                      default="localhost",
19 
                      type="string", help="The HOST where the mongo server is running",
20 
                      metavar="mongo_host")
13755 kshitij.so 21 
 
14258 kshitij.so 22 
(options, args) = parser.parse_args()
14325 kshitij.so 23 
 
24 
mc = MemCache(options.mongoHost)
25 
 
14258 kshitij.so 26 
DataService.initialize(db_hostname=options.hostname)
27 
 
13755 kshitij.so 28 
con = None
15907 kshitij.so 29 
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
13755 kshitij.so 30 
DISCOUNT_TYPE = {'MRP':1,'DP':2}
31 
LATEST_UPDATED_ITEMS = []
32 
 
33 
now = datetime.now()
34 
 
35 
class __SkuInfo:
36 
 
37 
    def __init__(self, _id, skuBundleId, category_id, mrp, available_price, source_id, rank, maxNlc, minNlc, schemeAmount, minDiscount, \
16019 kshitij.so 38 
                 maxDiscount, discountType, dp, nlcPoints, status, in_stock, maxprice, brand, dealType, dealPoints, brand_id, manualDealThresholdPrice,\
39 
                 codAvailable):
13755 kshitij.so 40 
        self._id = _id
41 
        self.skuBundleId = skuBundleId
42 
        self.category_id = category_id
43 
        self.mrp = mrp
44 
        self.available_price = available_price
45 
        self.source_id = source_id
46 
        self.rank = rank
47 
        self.maxNlc = maxNlc
48 
        self.minNlc = minNlc
49 
        self.schemeAmount = schemeAmount
50 
        self.minDiscount = minDiscount
51 
        self.maxDiscount = maxDiscount
52 
        self.discountType = discountType
53 
        self.dp = dp
54 
        self.nlcPoints = nlcPoints
13824 kshitij.so 55 
        self.status = status
56 
        self.in_stock = in_stock
57 
        self.maxprice = maxprice
14035 kshitij.so 58 
        self.brand = brand
14306 kshitij.so 59 
        self.dealType = dealType
14388 kshitij.so 60 
        self.dealPoints = dealPoints
15063 kshitij.so 61 
        self.brand_id = brand_id
15271 kshitij.so 62 
        self.manualDealThresholdPrice = manualDealThresholdPrice
16019 kshitij.so 63 
        self.codAvailable = codAvailable
13755 kshitij.so 64 
 
65 
 
14258 kshitij.so 66 
def get_mongo_connection(host=options.mongoHost, port=27017):
13755 kshitij.so 67 
    global con
68 
    if con is None:
69 
        print "Establishing connection %s host and port %d" %(host,port)
70 
        try:
71 
            con = pymongo.MongoClient(host, port)
72 
        except Exception, e:
73 
            print e
74 
            return None
75 
    return con
76 
 
77 
def populateStuff():
78 
    print "Inside populate"
79 
    global LATEST_UPDATED_ITEMS
80 
    """Fetch latest updated items across portals
81 
       and calculate max and min R-Nlc"""
82 
    offset= 0
83 
    while(True):
84 
        print "Fetching records offset %d and limit %d" %(offset,300)
85 
        topSkus = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'updatedOn': { "$gt": to_java_date(now - timedelta(hours=4))} }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).skip(offset).limit(300))
86 
        if len((topSkus)) == 0:
87 
            break
88 
        #topSkus = collection.find( {'_id':664})
89 
        for sku in topSkus:
13908 kshitij.so 90 
            """Fix this """
91 
            #TODO Compute deal flags else where.
13755 kshitij.so 92 
            info = __SkuInfo(sku['_id'], sku['skuBundleId'], sku['category_id'], sku['mrp'], sku['available_price'], sku['source_id'], sku['rank'], None, None, 0.0, None, \
16019 kshitij.so 93 
                             None, None, None, None, sku['status'], sku['in_stock'],sku['maxPrice'],sku['brand'].strip().upper(), 0, 0, sku['brand_id'], None, sku['codAvailable'])
15854 kshitij.so 94 
            exceptionalNlc = list(get_mongo_connection().Catalog.ExceptionalNlc.find( {"$and" : [ {'skuBundleId':info.skuBundleId}, {'overrideNlc':1} ]} ))
13755 kshitij.so 95 
            if len(exceptionalNlc) > 0:
96 
                """Exceptional nlc found, no need to calculate max and min R-nlc"""
97 
                info.maxNlc = exceptionalNlc[0]['maxNlc']
98 
                info.minNlc = exceptionalNlc[0]['minNlc']
14842 kshitij.so 99 
                if info.maxprice == 0:
100 
                    info.maxprice = exceptionalNlc[0]['maxNlc']
13755 kshitij.so 101 
                LATEST_UPDATED_ITEMS.append(info)
102 
                continue
103 
 
15854 kshitij.so 104 
            skuSchemeDetails = list(get_mongo_connection().Catalog.SkuSchemeDetails.find( {'skuBundleId':info.skuBundleId}))
13755 kshitij.so 105 
            if len(skuSchemeDetails) > 0:
106 
                """Sku scheme details, populate scheme amount (Recently added)"""
13824 kshitij.so 107 
 
108 
                #TODO Add start date and end date of scehems
109 
 
13755 kshitij.so 110 
                info.schemeAmount = float(skuSchemeDetails[0]['schemeAmount'])
111 
 
15854 kshitij.so 112 
            skuDealerPrices = list(get_mongo_connection().Catalog.SkuDealerPrices.find( {'skuBundleId':info.skuBundleId} ) )
13755 kshitij.so 113 
            if len(skuDealerPrices) > 0:
114 
                info.dp = skuDealerPrices[0]['dp']
15854 kshitij.so 115 
            skuDiscount = list(get_mongo_connection().Catalog.SkuDiscountInfo.find( {'skuBundleId':info.skuBundleId} ) )
13755 kshitij.so 116 
            if len(skuDiscount) > 0:
117 
                """Sku rule found, populate max , min Discount and discount type"""
118 
                info.maxDiscount = skuDiscount[0]['max_discount']
119 
                info.minDiscount = skuDiscount[0]['min_discount']
120 
                info.discountType = DISCOUNT_TYPE.get(skuDiscount[0]['discountType'].upper())
121 
                LATEST_UPDATED_ITEMS.append(info)
122 
                continue
123 
 
14035 kshitij.so 124 
            categoryDiscount = list(get_mongo_connection().Catalog.CategoryDiscount.find( {"$and" : [{'brand':sku['brand'].strip().upper()}, {'category_id':sku['category_id']} ]} ))
13755 kshitij.so 125 
            if len(categoryDiscount) > 0:
126 
                info.maxDiscount = categoryDiscount[0]['max_discount']
127 
                info.minDiscount = categoryDiscount[0]['min_discount']
128 
                info.discountType = DISCOUNT_TYPE.get(categoryDiscount[0]['discountType'].upper())
129 
 
130 
            LATEST_UPDATED_ITEMS.append(info)
131 
        offset = offset + 300
132 
    for lol in LATEST_UPDATED_ITEMS:
133 
        print lol.__dict__
134 
 
135 
 
136 
def calculateNlc():
137 
    global LATEST_UPDATED_ITEMS
138 
    populated = 0
139 
    while(populated <= len(LATEST_UPDATED_ITEMS)):
140 
        inventory_client = InventoryClient().get_client()
141 
        for obj in LATEST_UPDATED_ITEMS[populated:300+populated]:
142 
            if obj.maxNlc > 0 and obj.minNlc > 0:
143 
                continue
144 
            saholic_sku = list(get_mongo_connection().Catalog.MasterData.find( {"$and":[{'skuBundleId': obj.skuBundleId}, { 'source_id' : SOURCE_MAP.get('SAHOLIC')}] }))
145 
            identifier = None
146 
            if len(saholic_sku) > 0:
147 
                identifier = saholic_sku[0]['identifier']
148 
            if obj.discountType == DISCOUNT_TYPE.get('MRP'):
149 
                if obj.mrp == 0:
150 
                    """Now mrp is zero, so we have to use saholic MRP"""
151 
                    if identifier is not None:
152 
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
153 
                        obj.mrp = it.mrp
154 
                if obj.mrp > 0:
14842 kshitij.so 155 
                    print obj._id
13755 kshitij.so 156 
                    obj.minNlc = obj.mrp - (obj.mrp * obj.maxDiscount/100) - obj.schemeAmount
13824 kshitij.so 157 
                    obj.maxNlc = obj.mrp - (obj.mrp * obj.minDiscount/100) - obj.schemeAmount
14842 kshitij.so 158 
                    if obj.maxprice == 0:
159 
                        obj.maxprice = obj.maxNlc
13755 kshitij.so 160 
            elif obj.discountType == DISCOUNT_TYPE.get('DP'):
161 
                if obj.dp == 0:
162 
                    """Now dp is zero, so we have to use saholic minimum dp for item"""
163 
                    if identifier is not None:
164 
                        it = Item.query.filter_by(catalog_item_id=identifier).first()
165 
                        try:
166 
                            vendorPricing = inventory_client.getAllItemPricing(it.id)
167 
                            min_dp = min(pricing.dealerPrice for pricing in vendorPricing)
168 
                            obj.dp = min_dp
169 
                        except:
170 
                            pass
171 
                if obj.dp > 0:
172 
                    obj.minNlc = obj.dp - (obj.dp * obj.maxDiscount/100) - obj.schemeAmount
13824 kshitij.so 173 
                    obj.maxNlc = obj.dp - (obj.dp * obj.minDiscount/100) - obj.schemeAmount
14842 kshitij.so 174 
                    if obj.maxprice == 0:
175 
                        obj.maxprice = obj.maxNlc
13755 kshitij.so 176 
            else:
177 
                """No rule found, use saholic min nlc as max and min R-Nlc"""
178 
                if identifier is not None:
179 
                    it = Item.query.filter_by(catalog_item_id=identifier).first()
180 
                    try:
181 
                        vendorPricing = inventory_client.getAllItemPricing(it.id)
182 
                        min_nlc = min(pricing.nlc for pricing in vendorPricing)
183 
                        obj.maxNlc = min_nlc
13824 kshitij.so 184 
                        obj.minNlc = min_nlc
14842 kshitij.so 185 
                        if obj.maxprice == 0:
186 
                            obj.maxprice = obj.maxNlc
13755 kshitij.so 187 
                    except:
188 
                        pass
189 
        populated = populated + 300
190 
        time.sleep(10)
191 
 
14306 kshitij.so 192 
def addManualDealsInfo():
193 
    for sku in LATEST_UPDATED_ITEMS:
194 
        manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':sku.source_id, 'sku':sku._id}))
195 
        if len(manualDeal) > 0:
196 
            sku.dealType = manualDeal[0]['dealType']
15271 kshitij.so 197 
            if sku.available_price <= manualDeal[0]['dealThresholdPrice']:
198 
                sku.dealPoints = manualDeal[0]['dealPoints']
199 
                sku.manualDealThresholdPrice = manualDeal[0]['dealThresholdPrice']
200 
            else:
201 
                sku.dealPoints = 0
14310 kshitij.so 202 
 
203 
    """Remove deal flag from expired deals"""
14311 kshitij.so 204 
    manualDeals = list(get_mongo_connection().Catalog.Deals.find({'dealType':1}))
14310 kshitij.so 205 
    for manualDeal in manualDeals:
14389 kshitij.so 206 
        d_manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':manualDeal['source_id'], 'sku':manualDeal['_id']}))
207 
        if len(d_manualDeal) > 0:
14310 kshitij.so 208 
            continue
209 
        else:
15271 kshitij.so 210 
            get_mongo_connection().Catalog.Deals.update({'_id':manualDeal['_id']},{"$set":{'dealType':0,'dealPoints':0,'manualDealThresholdPrice':None}},upsert=False, multi=False)
14310 kshitij.so 211 
 
14306 kshitij.so 212 
 
13755 kshitij.so 213 
def calculateNlcPoints():
214 
    global LATEST_UPDATED_ITEMS
14842 kshitij.so 215 
    print "inside nlc oints"
13755 kshitij.so 216 
    for sku in LATEST_UPDATED_ITEMS:
217 
        if sku.maxNlc and sku.minNlc:
14842 kshitij.so 218 
            print sku._id
13824 kshitij.so 219 
 
220 
            """Create map - TODO"""
221 
 
222 
            if sku.status == 2:
223 
                eolWeight = .60
224 
            else:
225 
                eolWeight = 1.0
226 
            if sku.category_id == 3:
227 
                basePointPercentage = 5.0
228 
                maxNlcPoints = 200
229 
            elif sku.category_id == 5:
230 
                basePointPercentage = 8.0
231 
                maxNlcPoints = 150
232 
            else:
233 
                basePointPercentage = 10.0
234 
                maxNlcPoints = 150
14842 kshitij.so 235 
            discFromMinNlc = float((sku.minNlc - sku.available_price))/sku.available_price *100
236 
            discFromMaxNlc = float((sku.maxNlc - sku.available_price))/sku.available_price *100
237 
            print discFromMinNlc
238 
            print discFromMaxNlc
13755 kshitij.so 239 
            if discFromMinNlc > 0:
13824 kshitij.so 240 
                nlcPoints = 100/basePointPercentage * discFromMinNlc
13755 kshitij.so 241 
            elif discFromMinNlc < 0 and discFromMaxNlc > 0:
242 
                nlcPoints = 0
243 
            else:
13824 kshitij.so 244 
                nlcPoints = 100/basePointPercentage * discFromMinNlc
245 
            if (min(nlcPoints,maxNlcPoints)) > 0:
246 
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints)) * eolWeight
247 
            else:
248 
                sku.nlcPoints = (min(nlcPoints,maxNlcPoints))
13755 kshitij.so 249 
        else:
250 
            sku.nlcPoints = 0
251 
 
252 
def commitData():
253 
    global LATEST_UPDATED_ITEMS
254 
    for sku in LATEST_UPDATED_ITEMS:
255 
        #get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{'$set' : sku.__dict__},upsert=True,multi=True)
14020 amit.gupta 256 
        get_mongo_connection().Catalog.Deals.update({'_id':sku._id},{"$set":sku.__dict__},upsert=True)
13755 kshitij.so 257 
 
258 
 
259 
def addBestSellerPoints():
260 
    allItems = list(get_mongo_connection().Catalog.Deals.find({}))
261 
    for sku in allItems:
13824 kshitij.so 262 
        bestSellerPoints = list(get_mongo_connection().Catalog.BestSellerPoints.find( {"$and":[{'min_rank': { "$lte": sku['rank'] } }, {'max_rank': { "$gte": sku['rank'] } } , { 'category_id' : sku['category_id'] }, { 'source_id' : sku['source_id'] }] } ))
13755 kshitij.so 263 
        if len(bestSellerPoints) > 0:
13824 kshitij.so 264 
            print bestSellerPoints[0]['points']
265 
            if (bestSellerPoints[0]['points']) > 0:
266 
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points']) * bestSellerPoints[0]['weightage']
267 
            else:
268 
                sku['bestSellerPoints'] = (bestSellerPoints[0]['points'])
13755 kshitij.so 269 
        else:
13824 kshitij.so 270 
            sku['bestSellerPoints'] = -100
14114 kshitij.so 271 
        #sku['totalPoints'] = sku['bestSellerPoints'] + sku['nlcPoints']
272 
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'bestSellerPoints':sku['bestSellerPoints']}},multi=False)
273 
 
14390 kshitij.so 274 
    shortageSkus = get_mongo_connection().Catalog.MasterData.find({"$and":[{'is_shortage': 1 }, { 'source_id' : { "$in": SOURCE_MAP.values() } }] }).distinct('_id')
275 
    print "Shortage skus"
276 
    print shortageSkus
14388 kshitij.so 277 
 
14114 kshitij.so 278 
    for sku in allItems:
279 
        deal_item = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':sku['skuBundleId']}).sort('bestSellerPoints',pymongo.DESCENDING).limit(1))
14142 kshitij.so 280 
        sku['catalogBestSellerPoints'] = deal_item[0]['bestSellerPoints']
14388 kshitij.so 281 
        shortagePoints = 50 if sku['_id'] in shortageSkus else 0
14391 kshitij.so 282 
        print "Shortage points for ",sku['_id']
283 
        print shortagePoints
15271 kshitij.so 284 
        sku['totalPoints'] = sku['catalogBestSellerPoints'] + sku['nlcPoints'] + shortagePoints + sku['dealPoints']
14114 kshitij.so 285 
        get_mongo_connection().Catalog.Deals.update({'_id':sku['_id']},{'$set':{'catalogBestSellerPoints':sku['catalogBestSellerPoints'],'totalPoints':sku['totalPoints']}},multi=False)
15271 kshitij.so 286 
 
287 
 
13755 kshitij.so 288 
 
14325 kshitij.so 289 
def populateNegativeDeals():
290 
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
291 
    mc.set("negative_deals", negativeDeals, 600)
292 
 
13828 kshitij.so 293 
def elimiateSimilarDeals():
13912 kshitij.so 294 
    allItems = get_mongo_connection().Catalog.Deals.find().distinct('skuBundleId')
295 
    for skuBundleId in allItems:
296 
        print skuBundleId
297 
        similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
13828 kshitij.so 298 
        bestPrice = float("inf")
299 
        bestOne = None
300 
        bestSellerPoints = 0
301 
        toUpdate = []
16019 kshitij.so 302 
        prepaidBestPrice = float("inf")
303 
        prepaidBestOne = None
304 
        prepaidBestSellerPoints = 0
13828 kshitij.so 305 
        for similarItem in similarItems:
16019 kshitij.so 306 
            if similarItem['codAvailable'] ==1:
307 
                if mc.get("negative_deals") is None:
308 
                    populateNegativeDeals()
309 
                if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
310 
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
311 
                    continue
312 
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
313 
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
314 
                    continue
315 
                if similarItem['available_price'] < bestPrice:
316 
                    bestOne = similarItem
317 
                    bestPrice = similarItem['available_price']
318 
                    bestSellerPoints = similarItem['bestSellerPoints']
319 
                elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
320 
                    bestOne = similarItem
321 
                    bestPrice = similarItem['available_price']
322 
                    bestSellerPoints = similarItem['bestSellerPoints']
323 
                else:
324 
                    pass
13828 kshitij.so 325 
            else:
16019 kshitij.so 326 
                if mc.get("negative_deals") is None:
327 
                    populateNegativeDeals()
328 
                if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
329 
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
330 
                    continue
331 
                if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
332 
                    get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
333 
                    continue
334 
                if similarItem['available_price'] < prepaidBestPrice:
335 
                    prepaidBestOne = similarItem
336 
                    prepaidBestPrice = similarItem['available_price']
337 
                    prepaidBestSellerPoints = similarItem['bestSellerPoints']
338 
                elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
339 
                    prepaidBestOne = similarItem
340 
                    prepaidBestPrice = similarItem['available_price']
341 
                    prepaidBestSellerPoints = similarItem['bestSellerPoints']
342 
                else:
343 
                    pass
16026 kshitij.so 344 
        if bestOne is not None or prepaidBestOne is not None:
13828 kshitij.so 345 
            for similarItem in similarItems:
346 
                toUpdate.append(similarItem['_id'])
16026 kshitij.so 347 
            if bestOne is not None:
348 
                toUpdate.remove(bestOne['_id'])
349 
                get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
350 
            if prepaidBestOne is not None:
351 
                toUpdate.remove(prepaidBestOne['_id'])
352 
                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
13828 kshitij.so 353 
        if len(toUpdate) > 0:
16019 kshitij.so 354 
            get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
13755 kshitij.so 355 
 
356 
def main():
14263 kshitij.so 357 
    try:
358 
        populateStuff()
359 
        calculateNlc()
14306 kshitij.so 360 
        addManualDealsInfo()
14263 kshitij.so 361 
    finally:
362 
        session.close()
13755 kshitij.so 363 
    calculateNlcPoints()
364 
    commitData()
365 
    addBestSellerPoints()
13828 kshitij.so 366 
    elimiateSimilarDeals()
13755 kshitij.so 367 
 
13828 kshitij.so 368 
 
13755 kshitij.so 369 
if __name__=='__main__':
14842 kshitij.so 370 
    main()