Subversion Repositories SmartDukaan

Rev

Rev 17076 | Rev 19191 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 17076 Rev 17107
Line 27... Line 27...
27 
 
 27 
 
28 
mc = MemCache(options.mongoHost)
 28 
mc = MemCache(options.mongoHost)
29 
 
 29 
 
30 
def populate():
 30 
def populate():
31 
    toScrapMap = {}
 31 
    toScrapMap = {}
32 
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
 32 
    bestSellers = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'rank':{'$gt':0}}))
33 
    for bestSeller in bestSellers:
33 
    for bestSeller in bestSellers:
34 
        snapdealBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':7}))
 34 
        snapdealBestSellers = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':7}))
35 
        for data in snapdealBestSellers:
 35 
        for data in snapdealBestSellers:
36 
            if not toScrapMap.has_key(data['_id']):
 36 
            if not toScrapMap.has_key(data['_id']):
37 
                data['dealFlag'] = 0
 37 
                data['dealFlag'] = 0
38 
                data['dealType'] = 0
 38 
                data['dealType'] = 0
39 
                toScrapMap[data['_id']] = data
 39 
                toScrapMap[data['_id']] = data
40 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':7,'showDeal':1,'totalPoints':{'$gt':-100}}))
 40 
    dealFlagged = list(get_mongo_connection(host=options.mongoHost).Catalog.Deals.find({'source_id':7,'showDeal':1,'totalPoints':{'$gt':-100}}))
41 
    for deal in dealFlagged:
 41 
    for deal in dealFlagged:
42 
        if not toScrapMap.has_key(deal['_id']):
 42 
        if not toScrapMap.has_key(deal['_id']):
43 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
 43 
            data = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'_id':deal['_id']}))
44 
            data[0]['dealFlag'] = 0
 44 
            data[0]['dealFlag'] = 0
45 
            data[0]['dealType'] = 0
 45 
            data[0]['dealType'] = 0
46 
            toScrapMap[deal['_id']] = data[0]
 46 
            toScrapMap[deal['_id']] = data[0]
47 
    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':7}))
 47 
    manualDeals = list(get_mongo_connection(host=options.mongoHost).Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':7}))
48 
    for manualDeal in manualDeals:
 48 
    for manualDeal in manualDeals:
49 
        if not toScrapMap.has_key(manualDeal['sku']):
 49 
        if not toScrapMap.has_key(manualDeal['sku']):
50 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
 50 
            data = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'_id':manualDeal['sku']}))
51 
            if len(data) > 0:
 51 
            if len(data) > 0:
52 
                data[0]['dealFlag'] = 1
 52 
                data[0]['dealFlag'] = 1
53 
                data[0]['dealType'] = manualDeal['dealType']
 53 
                data[0]['dealType'] = manualDeal['dealType']
54 
                toScrapMap[manualDeal['sku']] = data[0]
 54 
                toScrapMap[manualDeal['sku']] = data[0]
55 
        else:
 55 
        else:
Line 83... Line 83...
83 
        url = 'http://m.homeshop18.com/product.mobi?productId='+str(data['identifier'])
 83 
        url = 'http://m.homeshop18.com/product.mobi?productId='+str(data['identifier'])
84 
        scraper = HomeShop18Scraper.HomeShop18Scraper()
 84 
        scraper = HomeShop18Scraper.HomeShop18Scraper()
85 
        result = scraper.read(url)
 85 
        result = scraper.read(url)
86 
    except:
 86 
    except:
87 
        print "Unable to scrape %d"%(data['_id'])
 87 
        print "Unable to scrape %d"%(data['_id'])
88 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now())}})
 88 
        get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now())}})
89 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})
 89 
        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})
90 
        return
 90 
        return
91
91
92 
    inStock = 0
 92 
    inStock = 0
93
93
94 
    lowestOfferPrice = 0
 94 
    lowestOfferPrice = 0
Line 102... Line 102...
102
102
103 
    if lowestOfferPrice ==0:
 103 
    if lowestOfferPrice ==0:
104 
        inStock = 0
 104 
        inStock = 0
105
105
106 
    if inStock  == 1:
 106 
    if inStock  == 1:
107 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
 107 
        get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
108 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
 108 
        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
109 
    else:
 109 
    else:
110 
        lowestOfferPrice = data['available_price']
 110 
        lowestOfferPrice = data['available_price']
111 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
 111 
        get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
112 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
 112 
        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable']}}, multi=True)
113
113
114 
    try:
 114 
    try:
115 
        recomputeDeal(data)
 115 
        recomputeDeal(data)
116 
    except:
 116 
    except:
117 
        print "Unable to compute deal for ",data['skuBundleId']
 117 
        print "Unable to compute deal for ",data['skuBundleId']
118
118
119 
def populateNegativeDeals():
 119 
def populateNegativeDeals():
120 
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
 120 
    negativeDeals = get_mongo_connection(host=options.mongoHost).Catalog.NegativeDeals.find().distinct('sku')
121 
    mc.set("negative_deals", negativeDeals, 600)
 121 
    mc.set("negative_deals", negativeDeals, 600)
122
122
123 
def recomputeDeal(item):
 123 
def recomputeDeal(item):
124 
    """Lets recompute deal for this bundle"""
 124 
    """Lets recompute deal for this bundle"""
125 
    print "Recomputing for bundleId %d" %(item.get('skuBundleId'))
 125 
    print "Recomputing for bundleId %d" %(item.get('skuBundleId'))
126 
    skuBundleId = item['skuBundleId']
 126 
    skuBundleId = item['skuBundleId']
127
127
128 
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
 128 
    similarItems = list(get_mongo_connection(host=options.mongoHost).Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
129 
    bestPrice = float("inf")
 129 
    bestPrice = float("inf")
130 
    bestOne = None
 130 
    bestOne = None
131 
    bestSellerPoints = 0
 131 
    bestSellerPoints = 0
132 
    toUpdate = []
 132 
    toUpdate = []
133 
    prepaidBestPrice = float("inf")
 133 
    prepaidBestPrice = float("inf")
Line 136... Line 136...
136 
    for similarItem in similarItems:
 136 
    for similarItem in similarItems:
137 
        if similarItem['codAvailable'] ==1:
 137 
        if similarItem['codAvailable'] ==1:
138 
            if mc.get("negative_deals") is None:
 138 
            if mc.get("negative_deals") is None:
139 
                populateNegativeDeals()
 139 
                populateNegativeDeals()
140 
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
 140 
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
141 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
 141 
                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
142 
                continue
 142 
                continue
143 
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
 143 
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
144 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
 144 
                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
145 
                continue
 145 
                continue
146 
            if similarItem['available_price'] < bestPrice:
 146 
            if similarItem['available_price'] < bestPrice:
147 
                bestOne = similarItem
 147 
                bestOne = similarItem
148 
                bestPrice = similarItem['available_price']
 148 
                bestPrice = similarItem['available_price']
149 
                bestSellerPoints = similarItem['bestSellerPoints']
 149 
                bestSellerPoints = similarItem['bestSellerPoints']
Line 155... Line 155...
155 
                pass
 155 
                pass
156 
        else:
 156 
        else:
157 
            if mc.get("negative_deals") is None:
 157 
            if mc.get("negative_deals") is None:
158 
                populateNegativeDeals()
 158 
                populateNegativeDeals()
159 
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
 159 
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
160 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
 160 
                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
161 
                continue
 161 
                continue
162 
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
 162 
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
163 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
 163 
                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
164 
                continue
 164 
                continue
165 
            if similarItem['source_id'] == SOURCE_MAP.get('PAYTM.COM'):
 165 
            if similarItem['source_id'] == SOURCE_MAP.get('PAYTM.COM'):
166 
                similarItem['available_price'] = similarItem['gross_price']
 166 
                similarItem['available_price'] = similarItem['gross_price']
167 
            if similarItem['available_price'] < prepaidBestPrice:
 167 
            if similarItem['available_price'] < prepaidBestPrice:
168 
                prepaidBestOne = similarItem
 168 
                prepaidBestOne = similarItem
Line 177... Line 177...
177 
    if bestOne is not None or prepaidBestOne is not None:
 177 
    if bestOne is not None or prepaidBestOne is not None:
178 
        for similarItem in similarItems:
 178 
        for similarItem in similarItems:
179 
            toUpdate.append(similarItem['_id'])
 179 
            toUpdate.append(similarItem['_id'])
180 
        if bestOne is not None:
 180 
        if bestOne is not None:
181 
            toUpdate.remove(bestOne['_id'])
 181 
            toUpdate.remove(bestOne['_id'])
182 
            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
 182 
            get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
183 
        if prepaidBestOne is not None:
 183 
        if prepaidBestOne is not None:
184 
            if bestOne is not None:
 184 
            if bestOne is not None:
185 
                if prepaidBestOne['available_price'] < bestOne['available_price']:
185 
                if prepaidBestOne['available_price'] < bestOne['available_price']:
186 
                    toUpdate.remove(prepaidBestOne['_id'])
 186 
                    toUpdate.remove(prepaidBestOne['_id'])
187 
                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
 187 
                    get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
188 
            else:
 188 
            else:
189 
                toUpdate.remove(prepaidBestOne['_id'])
 189 
                toUpdate.remove(prepaidBestOne['_id'])
190 
                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
 190 
                get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
191 
    if len(toUpdate) > 0:
 191 
    if len(toUpdate) > 0:
192 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
 192 
        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
193 
 
 193 
 
194 
 
 194 
 
195 
def main():
 195 
def main():
196 
    populate()
 196 
    populate()
197 
 
 197