Subversion Repositories SmartDukaan

Rev

Rev 13866 | Rev 13919 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13829 kshitij.so 1 
import urllib2
2 
import simplejson as json
3 
import pymongo
4 
from dtr.utils.utils import to_java_date
13918 kshitij.so 5 
from datetime import datetime, timedelta
13829 kshitij.so 6 
from operator import itemgetter
7 
from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper
13830 kshitij.so 8 
from dtr.utils import FlipkartScraper
13829 kshitij.so 9 
 
10 
con = None
11 
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3}
12 
scraperFk = FlipkartScraper.FlipkartScraper()
13 
scraperAmazon = AmazonScraper()
14 
 
15 
headers = {
16 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
17 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
18 
            'Accept-Language' : 'en-US,en;q=0.8',
19 
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
20 
        }
21 
 
22 
def get_mongo_connection(host='localhost', port=27017):
23 
    global con
24 
    if con is None:
25 
        print "Establishing connection %s host and port %d" %(host,port)
26 
        try:
27 
            con = pymongo.MongoClient(host, port)
28 
        except Exception, e:
29 
            print e
30 
            return None
31 
    return con
32 
 
33 
def returnLatestPrice(data, source_id):
13918 kshitij.so 34 
    now = datetime.now()
13829 kshitij.so 35 
    if source_id == 1:
36 
        try:
13918 kshitij.so 37 
            if data['identifier'] is None or len(data['identifier'].strip())==0:
13829 kshitij.so 38 
                return {}
13918 kshitij.so 39 
 
40 
            try:
41 
                if data['updatedOn'] + timedelta(minutes=5) > to_java_date(now):
42 
                    print "sku id is already updated",data['_id']
43 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
44 
            except:
45 
                pass
46 
 
47 
 
13829 kshitij.so 48 
            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data['identifier'])
49 
            lowestPrice = 0.0
50 
            lowestPrice = scraperAmazon.read(url)
51 
            inStock = 0
52 
            if lowestPrice > 0:
53 
                inStock = 1
54 
            if lowestPrice > 0:
13918 kshitij.so 55 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
56 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock}}, multi=True)
13829 kshitij.so 57 
            else:
13918 kshitij.so 58 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0}}, multi=True)
59 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}}, multi=True)
60 
 
61 
            try:
62 
                recomputeDeal(data['skuBundleId'])
63 
            except:
64 
                print "Unable to compute deal for ",data['skuBundleId']
65 
 
13829 kshitij.so 66 
            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
67 
        except:
68 
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
69 
 
70 
    elif source_id ==3:
71 
        try:
13918 kshitij.so 72 
            if data['identifier'] is None or len(data['identifier'].strip())==0:
13829 kshitij.so 73 
                return {}
13918 kshitij.so 74 
 
75 
            try:
76 
                if data['updatedOn'] + timedelta(minutes=5) > to_java_date(now):
77 
                    print "sku id is already updated",data['_id']
78 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
79 
            except:
80 
                pass
81 
 
82 
 
13829 kshitij.so 83 
            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(data['identifier'])
84 
            req = urllib2.Request(url,headers=headers)
85 
            response = urllib2.urlopen(req)
86 
            json_input = response.read()
87 
            vendorInfo = json.loads(json_input)
88 
            lowestOfferPrice = 0
89 
            inStock = 0
90 
            for vendor in vendorInfo:
91 
                lowestOfferPrice = float(vendor['sellingPrice'])
92 
                stock = vendor['buyableInventory']
93 
                if stock > 0 and lowestOfferPrice > 0:
94 
                    inStock = 1
95 
                    break
96 
 
97 
            print lowestOfferPrice
98 
            print inStock
99 
            print "*************"
100 
            if inStock  == 1:
13918 kshitij.so 101 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
102 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock}}, multi=True)
13829 kshitij.so 103 
            else:
13918 kshitij.so 104 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
105 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
106 
 
107 
            try:
108 
                recomputeDeal(data['skuBundleId'])
109 
            except:
110 
                print "Unable to compute deal for ",data['skuBundleId']
111 
 
13829 kshitij.so 112 
            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
113 
        except:
114 
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
115 
 
116 
    elif source_id == 2:
117 
        try:
13918 kshitij.so 118 
            if data['identifier'] is None or len(data['identifier'].strip())==0:
13829 kshitij.so 119 
                return {}
13918 kshitij.so 120 
 
121 
            try:
122 
                if data['updatedOn'] + timedelta(minutes=5) > to_java_date(now):
123 
                    print "sku id is already updated",data['_id']
124 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
125 
            except:
126 
                pass
127 
 
128 
 
13829 kshitij.so 129 
            url = "http://www.flipkart.com/ps/%s"%(data['identifier'])
130 
            vendorsData = scraperFk.read(url)
131 
            sortedVendorsData = []
132 
            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
133 
            print "data",sortedVendorsData
13918 kshitij.so 134 
            lowestSp = 0
135 
            inStock = 0
13829 kshitij.so 136 
            for vData in sortedVendorsData:
13918 kshitij.so 137 
                lowestSp = vData['sellingPrice']
13829 kshitij.so 138 
                break
139 
            if lowestSp > 0:
140 
                inStock = 1
141 
            print lowestSp
142 
            print inStock
143 
            if lowestSp > 0:
13918 kshitij.so 144 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
145 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
13829 kshitij.so 146 
            else:
13918 kshitij.so 147 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
148 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
149 
 
150 
 
151 
            try:
152 
                recomputeDeal(data['skuBundleId'])
153 
            except:
154 
                print "Unable to compute deal for ",data['skuBundleId']
155 
 
13829 kshitij.so 156 
            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
157 
        except:
158 
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['inStock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
159 
    else:
160 
        return {}
161 
 
13918 kshitij.so 162 
 
163 
def recomputeDeal(skuBundleId):
164 
    """Lets recompute deal for this bundle"""
165 
    print "Recomputing for bundleId",skuBundleId
13829 kshitij.so 166 
 
13918 kshitij.so 167 
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
168 
    bestPrice = float("inf")
169 
    bestOne = None
170 
    bestSellerPoints = 0
171 
    toUpdate = []
172 
    for similarItem in similarItems:
173 
        if similarItem['in_stock'] == 0:
174 
            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
175 
            continue
176 
        if similarItem['available_price'] < bestPrice:
177 
            bestOne = similarItem
178 
            bestPrice = similarItem['available_price']
179 
            bestSellerPoints = similarItem['bestSellerPoints']
180 
        elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
181 
            bestOne = similarItem
182 
            bestPrice = similarItem['available_price']
183 
            bestSellerPoints = similarItem['bestSellerPoints']
184 
        else:
185 
            pass
186 
    if bestOne is not None:
187 
        for similarItem in similarItems:
188 
            toUpdate.append(similarItem['_id'])
189 
        toUpdate.remove(bestOne['_id'])
190 
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
191 
    if len(toUpdate) > 0:
192 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
13829 kshitij.so 193 
 
194 
def getLatestPrice(skuBundleId, source_id):
195 
    temp = []
196 
    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))
197 
    for item in itemIds:
198 
        temp.append(returnLatestPrice(item, source_id))
199 
    return temp
200 
 
13864 kshitij.so 201 
def getLatestPriceById(id):
202 
    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))
13866 kshitij.so 203 
    return returnLatestPrice(item[0], item[0]['source_id'])
13829 kshitij.so 204 
 
205 
 
206 
def main():
13866 kshitij.so 207 
    print getLatestPriceById(7)
13829 kshitij.so 208 
 
209 
if __name__=='__main__':
210 
    main()