Subversion Repositories SmartDukaan

Rev

Rev 14705 | Rev 15341 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14705 Rev 15267
Line 1... Line 1...
1 
import pymongo
 1 
import pymongo
2 
from dtr.utils.utils import to_java_date
 2 
from dtr.utils.utils import to_java_date, getNlcPoints
3 
from datetime import datetime, timedelta
 3 
from datetime import datetime, timedelta
4 
from dtr.utils import AmazonPriceOnlyScraper, AmazonDealScraper
 4 
from dtr.utils import AmazonPriceOnlyScraper, AmazonDealScraper
5 
from multiprocessing import Pool as ThreadPool
 5 
from multiprocessing import Pool as ThreadPool
6 
from multiprocessing import cpu_count
 6 
from multiprocessing import cpu_count
7 
import optparse
 7 
import optparse
8 
from dtr.storage.MemCache import MemCache
 8 
from dtr.storage.MemCache import MemCache
9 
from dtr.utils.utils import getCashBack
 9 
from dtr.utils.utils import getCashBack
- 
 
 10 
import traceback
10 
 
 11 
 
11 
con = None
 12 
con = None
12 
 
 13 
 
13 
parser = optparse.OptionParser()
 14 
parser = optparse.OptionParser()
14 
parser.add_option("-m", "--m", dest="mongoHost",
 15 
parser.add_option("-m", "--m", dest="mongoHost",
Line 36... Line 37...
36 
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
 37 
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
37 
    for bestSeller in bestSellers:
38 
    for bestSeller in bestSellers:
38 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
 39 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
39 
        for data in amazonBestSellers:
 40 
        for data in amazonBestSellers:
40 
            if not toScrapMap.has_key(data['_id']):
 41 
            if not toScrapMap.has_key(data['_id']):
- 
 
 42 
                data['dealFlag'] = 0
- 
 
 43 
                data['dealType'] = 0
- 
 
 44 
                data['dealPoints'] = 0
- 
 
 45 
                data['manualDealThresholdPrice'] = None
41 
                toScrapMap[data['_id']] = data
 46 
                toScrapMap[data['_id']] = data
42 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':0}}))
 47 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':0}}))
43 
    for deal in dealFlagged:
 48 
    for deal in dealFlagged:
44 
        if not toScrapMap.has_key(deal['_id']):
 49 
        if not toScrapMap.has_key(deal['_id']):
45 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
 50 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
- 
 
 51 
            data[0]['dealFlag'] = 0
- 
 
 52 
            data[0]['dealType'] = 0
- 
 
 53 
            data[0]['dealPoints'] = 0
- 
 
 54 
            data[0]['manualDealThresholdPrice'] = None
46 
            toScrapMap[deal['_id']] = data[0]
 55 
            toScrapMap[deal['_id']] = data[0]
47 
    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':1}))
 56 
    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':1}))
48 
    for manualDeal in manualDeals:
 57 
    for manualDeal in manualDeals:
49 
        if not toScrapMap.has_key(manualDeal['sku']):
 58 
        if not toScrapMap.has_key(manualDeal['sku']):
50 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
 59 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
51 
            if len(data) > 0:
 60 
            if len(data) > 0:
52 
                data[0]['dealFlag'] = 1
 61 
                data[0]['dealFlag'] = 1
53 
                data[0]['dealType'] = manualDeal['dealType']
 62 
                data[0]['dealType'] = manualDeal['dealType']
- 
 
 63 
                data[0]['dealPoints'] = manualDeal['dealPoints']
- 
 
 64 
                data[0]['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
54 
                toScrapMap[manualDeal['sku']] = data[0]
 65 
                toScrapMap[manualDeal['sku']] = data[0]
55 
        else:
 66 
        else:
56 
            data = toScrapMap.get(manualDeal['sku'])
 67 
            data = toScrapMap.get(manualDeal['sku'])
57 
            data['dealFlag'] = 1
 68 
            data['dealFlag'] = 1
58 
            data['dealType'] = manualDeal['dealType']
 69 
            data['dealType'] = manualDeal['dealType']
- 
 
 70 
            data['dealPoints'] = manualDeal['dealPoints']
- 
 
 71 
            data['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
59 
    pool = ThreadPool(cpu_count() * 2)
 72 
    pool = ThreadPool(cpu_count() * 2)
60 
    pool.map(scrapeAmazon,toScrapMap.values())
 73 
    pool.map(scrapeAmazon,toScrapMap.values())
61 
    pool.close()
 74 
    pool.close()
62 
    pool.join()
 75 
    pool.join()
63 
    print "joining threads at %s"%(str(datetime.now()))
 76 
    print "joining threads at %s"%(str(datetime.now()))
Line 112... Line 125...
112 
    else:
 125 
    else:
113 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
 126 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
114 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'dealType':data['dealType']}}, multi=True)
 127 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'dealType':data['dealType']}}, multi=True)
115
128
116 
    try:
 129 
    try:
117 
        recomputeDeal(data['skuBundleId'])
 130 
        recomputeDeal(data)
118 
    except:
 131 
    except:
119 
        print "Unable to compute deal for ",data['skuBundleId']
132 
        print "Unable to compute deal for ",data['skuBundleId']
120
133
121 
def populateNegativeDeals():
 134 
def populateNegativeDeals():
122 
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
 135 
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
123 
    mc.set("negative_deals", negativeDeals, 600)
136 
    mc.set("negative_deals", negativeDeals, 600)
124 
 
 137 
 
- 
 
 138 
def recomputePoints(item, deal):
- 
 
 139 
    try:
- 
 
 140 
        nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
- 
 
 141 
    except:
- 
 
 142 
        traceback.print_exc()
- 
 
 143 
        nlcPoints = deal['nlcPoints']
- 
 
 144 
    if item['manualDealThresholdPrice'] >= deal['available_price']:
- 
 
 145 
        dealPoints = item['dealPoints']
- 
 
 146 
    else:
- 
 
 147 
        dealPoints = 0
125 
def recomputeDeal(skuBundleId):
 148 
    get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
- 
 
 149 
 
- 
 
 150
- 
 
 151 
 
- 
 
 152 
def recomputeDeal(item):
126 
    """Lets recompute deal for this bundle"""
 153 
    """Lets recompute deal for this bundle"""
127 
    print "Recomputing for bundleId",skuBundleId
 154 
    print "Recomputing for bundleId",item.get('skuBundleId')
- 
 
 155 
    skuBundleId = item['skuBundleId']
128
156
129 
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
 157 
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
130 
    bestPrice = float("inf")
 158 
    bestPrice = float("inf")
131 
    bestOne = None
 159 
    bestOne = None
132 
    bestSellerPoints = 0
 160 
    bestSellerPoints = 0
133 
    toUpdate = []
 161 
    toUpdate = []
134 
    for similarItem in similarItems:
 162 
    for similarItem in similarItems:
135 
        if mc.get("negative_deals") is None:
 163 
        if mc.get("negative_deals") is None:
136 
            populateNegativeDeals()
 164 
            populateNegativeDeals()
137 
        try:
 165 
#        try:
138 
            cashBack = getCashBack(similarItem['_id'], similarItem['source_id'], similarItem['category_id'], mc, options.mongoHost)
 166 
#            cashBack = getCashBack(similarItem['_id'], similarItem['source_id'], similarItem['category_id'], mc, options.mongoHost)
139 
            if not cashBack or cashBack.get('cash_back_status')!=1:
 167 
#            if not cashBack or cashBack.get('cash_back_status')!=1:
140 
                pass
 168 
#                pass
141 
            else:
 169 
#            else:
142 
                if cashBack['cash_back_type'] ==1:
 170 
#                if cashBack['cash_back_type'] ==1:
143 
                    similarItem['available_price'] = similarItem['available_price'] - similarItem['available_price'] * float(cashBack['cash_back'])/100
 171 
#                    similarItem['available_price'] = similarItem['available_price'] - similarItem['available_price'] * float(cashBack['cash_back'])/100
144 
                elif cashBack['cash_back_type'] ==2:
 172 
#                elif cashBack['cash_back_type'] ==2:
145 
                    similarItem['available_price'] = similarItem['available_price'] - float(cashBack['cash_back'])
 173 
#                    similarItem['available_price'] = similarItem['available_price'] - float(cashBack['cash_back'])
146 
                else:
 174 
#                else:
147 
                    pass
 175 
#                    pass
148 
        except Exception as cashBackEx:
 176 
#        except Exception as cashBackEx:
149 
            print cashBackEx
 177 
#            print cashBackEx
150 
            print "Error calculating cashback."
 178 
#            print "Error calculating cashback."
- 
 
 179 
        if similarItem['_id'] == item['_id']:
- 
 
 180 
            try:
- 
 
 181 
                recomputePoints(item, similarItem)
- 
 
 182 
            except:
- 
 
 183 
                traceback.print_exc()
151 
        if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
 184 
        if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
152 
            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
 185 
            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
153 
            continue
 186 
            continue
154 
        if similarItem['available_price'] < bestPrice:
 187 
        if similarItem['available_price'] < bestPrice:
155 
            bestOne = similarItem
 188 
            bestOne = similarItem