Subversion Repositories SmartDukaan

Rev

Rev 14132 | Rev 14181 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14132 Rev 14180
Line 2... Line 2...
2 
import simplejson as json
 2 
import simplejson as json
3 
import pymongo
 3 
import pymongo
4 
from dtr.utils.utils import to_java_date
 4 
from dtr.utils.utils import to_java_date
5 
from datetime import datetime, timedelta
 5 
from datetime import datetime, timedelta
6 
import time
 6 
import time
- 
 
 7 
from multiprocessing import Pool as ThreadPool
- 
 
 8 
from multiprocessing import cpu_count
7 
 
 9 
 
8 
con = None
 10 
con = None
9 
 
 11 
 
10 
headers = {
12 
headers = {
11 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
 13 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
Line 23... Line 25...
23 
        except Exception, e:
 25 
        except Exception, e:
24 
            print e
 26 
            print e
25 
            return None
 27 
            return None
26 
    return con
 28 
    return con
27 
 
 29 
 
28 
def updatePrices():
 30 
def populate():
- 
 
 31 
    toScrapMap = {}
29 
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
 32 
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
30 
    for bestSeller in bestSellers:
33 
    for bestSeller in bestSellers:
31 
        snapdealBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':3}))
 34 
        snapdealBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':3}))
32 
        for data in snapdealBestSellers:
 35 
        for data in snapdealBestSellers:
- 
 
 36 
            if not toScrapMap.has_key(data['_id']):
- 
 
 37 
                toScrapMap[data['_id']] = data
- 
 
 38 
    for k, y in toScrapMap.iteritems():
- 
 
 39 
        print k,
- 
 
 40 
        print '\t',
- 
 
 41 
        print y
- 
 
 42 
    pool = ThreadPool(cpu_count() *2)
- 
 
 43 
    pool.map(updatePrices,toScrapMap.values())
- 
 
 44 
    pool.close()
- 
 
 45 
    pool.join()
- 
 
 46 
    print "joining threads"
- 
 
 47 
    print datetime.now()
- 
 
 48 
 
- 
 
 49 
 
- 
 
 50 
def updatePrices(data):
33 
            if data['source_id']!=3:
 51 
    if data['source_id']!=3:
34 
                continue
 52 
        return
35 
            print data['identifier']
 53 
    print data['identifier']
36 
            if data['identifier'] is None or len(data['identifier'].strip())==0:
 54 
    if data['identifier'] is None or len(data['identifier'].strip())==0:
37 
                print "continue"
 55 
        print "returning"
38 
                continue
 56 
        return
39
57
40 
            try:
 58 
    try:
41 
                if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
 59 
        if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
42 
                    print "sku id is already updated",data['_id']
60 
            print "sku id is already updated",data['_id']
43 
                    continue
 61 
            return
44 
            except:
 62 
    except:
45 
                pass
 63 
        pass
46
64
47 
            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(data['identifier'].strip())
 65 
    url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(data['identifier'].strip())
48 
            print url
 66 
    print url
49 
            time.sleep(1)
 67 
    time.sleep(1)
50 
            lowestOfferPrice = 0
 68 
    lowestOfferPrice = 0
51 
            instock = 0
 69 
    instock = 0
52 
            req = urllib2.Request(url,headers=headers)
 70 
    req = urllib2.Request(url,headers=headers)
53 
            response = urllib2.urlopen(req)
 71 
    response = urllib2.urlopen(req)
54 
            json_input = response.read()
 72 
    json_input = response.read()
55 
            if len(json_input) > 0:
 73 
    if len(json_input) > 0:
56 
                vendorInfo = json.loads(json_input)
 74 
        vendorInfo = json.loads(json_input)
57 
                for vendor in vendorInfo:
 75 
        for vendor in vendorInfo:
58 
                    lowestOfferPrice = float(vendor['sellingPrice'])
 76 
            lowestOfferPrice = float(vendor['sellingPrice'])
59 
                    stock = vendor['buyableInventory']
 77 
            stock = vendor['buyableInventory']
60 
                    if stock > 0 and lowestOfferPrice > 0:
 78 
            if stock > 0 and lowestOfferPrice > 0:
61 
                        instock = 1
 79 
                instock = 1
62 
                        break
 80 
                break
63
81
64 
            print lowestOfferPrice
 82 
    print lowestOfferPrice
65 
            print instock
 83 
    print instock
66 
            print stock
 84 
    print stock
67 
            print "*************"
 85 
    print "*************"
68 
            if instock  == 1:
 86 
    if instock  == 1:
69 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':instock}}, multi=True)
 87 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':instock}}, multi=True)
70 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':instock}}, multi=True)
 88 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':instock}}, multi=True)
71 
            else:
 89 
    else:
72 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':instock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
 90 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':instock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
73 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock}}, multi=True)
 91 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock}}, multi=True)
74
92
75 
            try:
 93 
    try:
76 
                recomputeDeal(data['skuBundleId'])
 94 
        recomputeDeal(data['skuBundleId'])
77 
            except:
 95 
    except:
78 
                print "Unable to compute deal for ",data['skuBundleId']
 96 
        print "Unable to compute deal for ",data['skuBundleId']
79
97
80
98
81 
def recomputeDeal(skuBundleId):
 99 
def recomputeDeal(skuBundleId):
82 
    """Lets recompute deal for this bundle"""
 100 
    """Lets recompute deal for this bundle"""
83 
    print "Recomputing for bundleId",skuBundleId
 101 
    print "Recomputing for bundleId",skuBundleId
84
102
85 
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
 103 
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
Line 108... Line 126...
108 
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
 126 
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
109 
    if len(toUpdate) > 0:
 127 
    if len(toUpdate) > 0:
110 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
 128 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
111
129
112 
def main():
 130 
def main():
113 
    updatePrices()
 131 
    populate()
114 
 
 132 
 
115 
if __name__=='__main__':
 133 
if __name__=='__main__':
116 
    main()
 134 
    main()
117 

            135