| 10503 |
kshitij.so |
1 |
import urllib2
|
|
|
2 |
from BeautifulSoup import BeautifulSoup
|
|
|
3 |
import re
|
|
|
4 |
|
|
|
5 |
class FlipkartScraper:
|
|
|
6 |
def __init__(self):
|
|
|
7 |
self.count_trials = 0
|
|
|
8 |
|
|
|
9 |
def read(self, url):
|
|
|
10 |
request = urllib2.Request(url)
|
|
|
11 |
request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')
|
|
|
12 |
opener = urllib2.build_opener()
|
|
|
13 |
response_data = ""
|
|
|
14 |
try:
|
|
|
15 |
response_data = opener.open(request).read()
|
|
|
16 |
|
|
|
17 |
except urllib2.HTTPError as e:
|
|
|
18 |
print 'ERROR: ', e
|
|
|
19 |
print 'Retrying'
|
|
|
20 |
self.count_trials += 1
|
|
|
21 |
|
|
|
22 |
if self.count_trials < 3:
|
|
|
23 |
return self.read(url)
|
|
|
24 |
|
|
|
25 |
self.response_data=response_data
|
|
|
26 |
|
|
|
27 |
def createData(self):
|
|
|
28 |
page=self.response_data.decode("utf-8")
|
|
|
29 |
self.soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
|
|
|
30 |
return self.scrape(self.soup)
|
|
|
31 |
|
|
|
32 |
|
|
|
33 |
def scrape(self,soup):
|
|
|
34 |
info = []
|
|
|
35 |
oddSeller = soup.findAll("div" , {"class" : "line seller-item odd "})
|
|
|
36 |
for data in oddSeller:
|
|
|
37 |
temp={}
|
|
|
38 |
price = data.find('span', attrs={'class' : re.compile('pxs-final-price.*')}).string.strip('Rs.').strip()
|
|
|
39 |
temp['sellingPrice']=float(price)
|
|
|
40 |
for sellerInfo in data.findAll("div",{"class":re.compile(".*seller-info*")}):
|
|
|
41 |
sellerName = sellerInfo.find('a').string
|
|
|
42 |
temp['sellerName'] = sellerName
|
|
|
43 |
for metrics in data.find("div",{"class":"fk-text-right"}):
|
|
|
44 |
try:
|
|
|
45 |
inputTags = metrics['data-lst-buytrend']
|
|
|
46 |
except TypeError:
|
|
|
47 |
continue
|
|
|
48 |
dataMetrics = metrics['data-listing-metrics']
|
|
|
49 |
try:
|
|
|
50 |
buyTrend = inputTags[0:str(inputTags).index('NWSR')].replace('_','')
|
|
|
51 |
except ValueError:
|
|
|
52 |
buyTrend = inputTags[0:str(inputTags).index('WSR')].replace('_','')
|
|
|
53 |
temp['buyTrend']=buyTrend
|
|
|
54 |
dataMetric = dataMetrics.split(';')
|
|
|
55 |
sellerCode = dataMetric[0]
|
|
|
56 |
temp['sellerCode']=sellerCode
|
|
|
57 |
temp['sellingPriceMetric'] = float(dataMetric[1])
|
|
|
58 |
temp['shippingTime'] = dataMetric[3]
|
|
|
59 |
temp['sellerScore'] = int(dataMetric[4])
|
|
|
60 |
info.append(temp)
|
|
|
61 |
evenSeller = soup.findAll("div" , {"class" : "line seller-item even "})
|
|
|
62 |
for data in evenSeller:
|
|
|
63 |
temp={}
|
|
|
64 |
price = data.find('span', attrs={'class' : re.compile('pxs-final-price.*')}).string.strip('Rs.')
|
|
|
65 |
temp['sellingPrice']=float(price)
|
|
|
66 |
for sellerInfo in data.findAll("div",{"class":re.compile(".*seller-info*")}):
|
|
|
67 |
sellerName = sellerInfo.find('a').string
|
|
|
68 |
temp['sellerName'] = sellerName
|
|
|
69 |
for metrics in data.find("div",{"class":"fk-text-right"}):
|
|
|
70 |
try:
|
|
|
71 |
inputTags = metrics['data-lst-buytrend']
|
|
|
72 |
except TypeError:
|
|
|
73 |
continue
|
|
|
74 |
dataMetrics = metrics['data-listing-metrics']
|
|
|
75 |
try:
|
|
|
76 |
buyTrend = inputTags[0:str(inputTags).index('NWSR')].replace('_','')
|
|
|
77 |
except ValueError:
|
|
|
78 |
buyTrend = inputTags[0:str(inputTags).index('WSR')].replace('_','')
|
|
|
79 |
temp['buyTrend']=buyTrend
|
|
|
80 |
dataMetric = dataMetrics.split(';')
|
|
|
81 |
temp['sellerCode'] = dataMetric[0]
|
|
|
82 |
temp['sellingPriceMetric'] = float(dataMetric[1])
|
|
|
83 |
temp['shippingTime'] = dataMetric[3]
|
|
|
84 |
temp['sellerScore'] = int(dataMetric[4])
|
|
|
85 |
info.append(temp)
|
|
|
86 |
return info
|
|
|
87 |
|
|
|
88 |
if __name__ == '__main__':
|
|
|
89 |
scraper = FlipkartScraper()
|
| 11193 |
kshitij.so |
90 |
scraper.read('http://www.flipkart.com/ps/MOBDPF4ZS43WXV3R')
|
| 10503 |
kshitij.so |
91 |
print scraper.createData()
|
|
|
92 |
|