| 14307 |
kshitij.so |
1 |
import urllib2
|
|
|
2 |
from BeautifulSoup import BeautifulSoup, NavigableString
|
|
|
3 |
import re
|
|
|
4 |
import sys
|
|
|
5 |
|
|
|
6 |
invalid_tags = ['b', 'i', 'u']
|
|
|
7 |
bestSellers = []
|
|
|
8 |
|
|
|
9 |
def strip_tags(html, invalid_tags):
|
|
|
10 |
soup = BeautifulSoup(html,convertEntities=BeautifulSoup.HTML_ENTITIES)
|
|
|
11 |
|
|
|
12 |
for tag in soup.findAll(True):
|
|
|
13 |
if tag.name in invalid_tags:
|
|
|
14 |
s = ""
|
|
|
15 |
|
|
|
16 |
for c in tag.contents:
|
|
|
17 |
if not isinstance(c, NavigableString):
|
|
|
18 |
c = strip_tags(unicode(c), invalid_tags)
|
|
|
19 |
s += unicode(c)
|
|
|
20 |
|
|
|
21 |
tag.replaceWith(s)
|
|
|
22 |
|
|
|
23 |
return soup
|
|
|
24 |
|
|
|
25 |
class AmazonScraper:
|
|
|
26 |
def __init__(self):
|
|
|
27 |
self.count_trials = 0
|
|
|
28 |
|
|
|
29 |
def read(self, url):
|
|
|
30 |
request = urllib2.Request(url)
|
|
|
31 |
request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')
|
|
|
32 |
response_data = ""
|
|
|
33 |
try:
|
|
|
34 |
response = urllib2.urlopen(request)
|
|
|
35 |
response_data = response.read()
|
|
|
36 |
response.close()
|
|
|
37 |
|
|
|
38 |
except urllib2.HTTPError as e:
|
|
|
39 |
print 'ERROR: ', e
|
|
|
40 |
print 'Retrying'
|
|
|
41 |
self.count_trials += 1
|
|
|
42 |
|
|
|
43 |
if self.count_trials < 3:
|
|
|
44 |
return self.read(url)
|
|
|
45 |
|
|
|
46 |
self.response_data=response_data
|
|
|
47 |
return self.createData()
|
|
|
48 |
|
|
|
49 |
def createData(self):
|
|
|
50 |
self.soup = strip_tags(self.response_data,invalid_tags)
|
|
|
51 |
self.response_data =None
|
|
|
52 |
return self.scrape(self.soup)
|
|
|
53 |
|
|
|
54 |
|
|
|
55 |
def scrape(self,soup):
|
|
|
56 |
try:
|
|
|
57 |
sellerData = soup.find("span" , {"id" : "priceblock_dealprice"})
|
|
|
58 |
dealPrice = float(sellerData.text.replace("Rs.","").replace(",",""))
|
|
|
59 |
except:
|
|
|
60 |
dealPrice = 0.0
|
|
|
61 |
try:
|
|
|
62 |
dealAvailablity = soup.find('div',{'id':'deal_availability'})
|
|
|
63 |
dealStatus = dealAvailablity.find('span',{'id':re.compile('dealStatusAvailability_*')})
|
|
|
64 |
dealStatus = float(dealStatus.text.replace("%","").replace(",",""))
|
|
|
65 |
except:
|
|
|
66 |
dealStatus = 100
|
|
|
67 |
|
|
|
68 |
if dealStatus < 100 and dealPrice > 0:
|
|
|
69 |
return dealPrice
|
|
|
70 |
else:
|
|
|
71 |
return 0.0
|
|
|
72 |
|
|
|
73 |
if __name__ == '__main__':
|
|
|
74 |
scraper = AmazonScraper()
|
|
|
75 |
print scraper.read('http://www.amazon.in/gp/product/B00FXLCLTO')
|
|
|
76 |
|