| 4039 |
varun.gupt |
1 |
'''
|
|
|
2 |
Created on 24-Aug-2011
|
|
|
3 |
|
|
|
4 |
@author: Varun Gupta
|
|
|
5 |
'''
|
|
|
6 |
|
|
|
7 |
from BeautifulSoup import BeautifulSoup
|
|
|
8 |
from BaseScraper import BaseScraper
|
|
|
9 |
|
|
|
10 |
class FlipcartScraper(BaseScraper):
|
|
|
11 |
|
|
|
12 |
def __init__(self):
|
|
|
13 |
BaseScraper.__init__(self)
|
|
|
14 |
self.url = None
|
|
|
15 |
self.id = None
|
|
|
16 |
|
|
|
17 |
def setUrl(self, url):
|
|
|
18 |
self.url = url
|
|
|
19 |
|
|
|
20 |
def scrape(self):
|
|
|
21 |
html = BaseScraper.read(self, self.url)
|
|
|
22 |
self.soup = BeautifulSoup(html)
|
|
|
23 |
self.phones = None
|
|
|
24 |
|
|
|
25 |
def getPhones(self):
|
|
|
26 |
phones = []
|
|
|
27 |
|
|
|
28 |
for div in self.soup.findAll('div', {'class': 'fk-product-thumb fkp-medium'}):
|
|
|
29 |
try:
|
|
|
30 |
anchor = div.findAll('a', {'class': 'title fk-anchor-link'})[0]
|
|
|
31 |
name = anchor['title'].strip()
|
|
|
32 |
price = None
|
|
|
33 |
product_url = anchor['href'].strip()
|
|
|
34 |
in_stock = 0 if div.findAll('b').__len__() > 0 else 1
|
|
|
35 |
|
|
|
36 |
for span in div.findAll('span'):
|
|
|
37 |
try:
|
|
|
38 |
if span['class'].find('price final-price') > -1:
|
|
|
39 |
price = span.string.strip()
|
|
|
40 |
except KeyError:
|
|
|
41 |
pass
|
|
|
42 |
try:
|
|
|
43 |
if price is None:
|
|
|
44 |
continue
|
|
|
45 |
else:
|
|
|
46 |
phones.append({'name': str(name), 'price': str(price), 'product_url': str(product_url), 'in_stock': in_stock})
|
|
|
47 |
|
|
|
48 |
except UnboundLocalError as e:
|
|
|
49 |
print e, name
|
|
|
50 |
print div
|
|
|
51 |
|
|
|
52 |
except UnicodeEncodeError as e:
|
|
|
53 |
print 'Unicode Error', e, name
|
|
|
54 |
name_ascii = "".join([char if ord(char) < 128 else " " for char in name])
|
|
|
55 |
print name_ascii
|
|
|
56 |
phones.append({"name": str(name_ascii), "price": str(price), "in_stock": in_stock, "product_url": str(product_url)})
|
|
|
57 |
except KeyError:
|
|
|
58 |
pass
|
|
|
59 |
self.phones = phones
|
|
|
60 |
return phones
|
|
|
61 |
|
|
|
62 |
def getNextUrl(self):
|
|
|
63 |
tab_info = self.soup.findAll('div', {'class': 'unit fk-lres-header-text'})[0]('b')
|
|
|
64 |
current_max = int(tab_info[0].string.split('-')[1])
|
|
|
65 |
total = int(tab_info[1].string)
|
|
|
66 |
|
|
|
67 |
if len(self.phones) > 0:
|
|
|
68 |
base_url = 'http://www.flipkart.com/mobiles/%s' % ('all/' if self.phones[0]['product_url'].find('/tablets/') == -1 else 'tablet-20278/')
|
|
|
69 |
|
|
|
70 |
if current_max < total:
|
|
|
71 |
return base_url + str(1 + (current_max / 20))
|
|
|
72 |
else:
|
|
|
73 |
return None
|
|
|
74 |
else:
|
|
|
75 |
return None
|
|
|
76 |
|
|
|
77 |
|
|
|
78 |
if __name__ == '__main__':
|
|
|
79 |
s = FlipcartScraper()
|
|
|
80 |
s.setUrl('http://www.flipkart.com/mobiles/all/27')
|
|
|
81 |
s.scrape()
|
|
|
82 |
phones = s.getPhones()
|
|
|
83 |
for p in phones: print p
|
|
|
84 |
print s.getNextUrl()
|