| 13754 |
kshitij.so |
1 |
import urllib2
|
|
|
2 |
import simplejson as json
|
|
|
3 |
import pymongo
|
|
|
4 |
from shop2020.model.v1.catalog.script.DataWeaveConsumer import __ProductInfo
|
|
|
5 |
|
|
|
6 |
headers = {
|
|
|
7 |
'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
|
|
|
8 |
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
|
9 |
'Accept-Language' : 'en-US,en;q=0.8',
|
|
|
10 |
'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
|
|
|
11 |
}
|
|
|
12 |
|
|
|
13 |
con = None
|
|
|
14 |
bestSellers = []
|
|
|
15 |
|
|
|
16 |
class __RankInfo:
|
|
|
17 |
|
|
|
18 |
def __init__(self, identifier, rank):
|
|
|
19 |
self.identifier = identifier
|
|
|
20 |
self.rank = rank
|
|
|
21 |
|
|
|
22 |
def get_mongo_connection(host='localhost', port=27017):
|
|
|
23 |
global con
|
|
|
24 |
if con is None:
|
|
|
25 |
print "Establishing connection %s host and port %d" %(host,port)
|
|
|
26 |
try:
|
|
|
27 |
con = pymongo.MongoClient(host, port)
|
|
|
28 |
except Exception, e:
|
|
|
29 |
print e
|
|
|
30 |
return None
|
|
|
31 |
return con
|
|
|
32 |
|
|
|
33 |
def scrapeBestSellerMobiles():
|
|
|
34 |
global bestSellers
|
|
|
35 |
rank = 1
|
|
|
36 |
for z in [0,20,40,60,80]:
|
|
|
37 |
url = "http://www.snapdeal.com/acors/json/product/get/search/175/%d/20?q=&sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
|
|
|
38 |
print url
|
|
|
39 |
request = urllib2.Request(url,headers=headers)
|
|
|
40 |
response = urllib2.urlopen(request)
|
|
|
41 |
|
|
|
42 |
json_input = response.read()
|
|
|
43 |
info = json.loads(json_input)
|
|
|
44 |
for offer in info['productOfferGroupDtos']:
|
|
|
45 |
for identifiers in offer['offers']:
|
|
|
46 |
r_info = __RankInfo((identifiers['supcs'])[0],rank)
|
|
|
47 |
bestSellers.append(r_info)
|
|
|
48 |
rank += 1
|
|
|
49 |
|
|
|
50 |
def scrapeBestSellerTablets():
|
|
|
51 |
global bestSellers
|
|
|
52 |
bestSellers = []
|
|
|
53 |
rank = 1
|
|
|
54 |
for z in [0,20,40,60,80]:
|
|
|
55 |
url = "http://www.snapdeal.com/acors/json/product/get/search/133/%d/20?sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
|
|
|
56 |
print url
|
|
|
57 |
request = urllib2.Request(url,headers=headers)
|
|
|
58 |
response = urllib2.urlopen(request)
|
|
|
59 |
|
|
|
60 |
json_input = response.read()
|
|
|
61 |
info = json.loads(json_input)
|
|
|
62 |
for offer in info['productOfferGroupDtos']:
|
|
|
63 |
for identifiers in offer['offers']:
|
|
|
64 |
r_info = __RankInfo((identifiers['supcs'])[0],rank)
|
|
|
65 |
bestSellers.append(r_info)
|
|
|
66 |
rank += 1
|
|
|
67 |
|
|
|
68 |
def resetRanks(category):
|
|
|
69 |
get_mongo_connection().Catalog.MasterData.update({'category':category,'source_id':3}, {'$set' : {'rank':0}}, multi=True)
|
|
|
70 |
|
|
|
71 |
def commitBestSellers():
|
|
|
72 |
print "Rank",
|
|
|
73 |
print '\t',
|
|
|
74 |
print 'Identifier'
|
|
|
75 |
for x in bestSellers:
|
|
|
76 |
print x.rank,
|
|
|
77 |
print '\t',
|
|
|
78 |
print x.identifier,
|
|
|
79 |
col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier.strip()})
|
|
|
80 |
print "count sku",
|
|
|
81 |
print '\t',
|
|
|
82 |
print len(list(col))
|
|
|
83 |
get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier.strip()}, {'$set' : {'rank':x.rank}}, multi=True)
|
|
|
84 |
|
|
|
85 |
def main():
|
|
|
86 |
scrapeBestSellerMobiles()
|
|
|
87 |
if len(bestSellers) > 0:
|
|
|
88 |
resetRanks('Mobiles')
|
|
|
89 |
commitBestSellers()
|
|
|
90 |
scrapeBestSellerTablets()
|
|
|
91 |
if len(bestSellers) > 0:
|
|
|
92 |
resetRanks('Tablets')
|
|
|
93 |
commitBestSellers()
|
|
|
94 |
|
|
|
95 |
if __name__=='__main__':
|
|
|
96 |
main()
|