| 13829 |
kshitij.so |
1 |
import urllib2
|
|
|
2 |
from BeautifulSoup import BeautifulSoup, NavigableString
|
|
|
3 |
import re
|
|
|
4 |
import sys
|
|
|
5 |
|
|
|
6 |
invalid_tags = ['b', 'i', 'u']
|
|
|
7 |
bestSellers = []
|
|
|
8 |
|
|
|
9 |
def strip_tags(html, invalid_tags):
|
|
|
10 |
soup = BeautifulSoup(html,convertEntities=BeautifulSoup.HTML_ENTITIES)
|
|
|
11 |
|
|
|
12 |
for tag in soup.findAll(True):
|
|
|
13 |
if tag.name in invalid_tags:
|
|
|
14 |
s = ""
|
|
|
15 |
|
|
|
16 |
for c in tag.contents:
|
|
|
17 |
if not isinstance(c, NavigableString):
|
|
|
18 |
c = strip_tags(unicode(c), invalid_tags)
|
|
|
19 |
s += unicode(c)
|
|
|
20 |
|
|
|
21 |
tag.replaceWith(s)
|
|
|
22 |
|
|
|
23 |
return soup
|
|
|
24 |
|
|
|
25 |
class AmazonScraper:
|
|
|
26 |
def __init__(self):
|
|
|
27 |
self.count_trials = 0
|
|
|
28 |
|
|
|
29 |
def read(self, url):
|
|
|
30 |
request = urllib2.Request(url)
|
|
|
31 |
request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')
|
|
|
32 |
response_data = ""
|
|
|
33 |
try:
|
| 14157 |
kshitij.so |
34 |
response = urllib2.urlopen(request)
|
|
|
35 |
response_data = response.read()
|
|
|
36 |
response.close()
|
| 13829 |
kshitij.so |
37 |
|
|
|
38 |
except urllib2.HTTPError as e:
|
|
|
39 |
print 'ERROR: ', e
|
|
|
40 |
print 'Retrying'
|
|
|
41 |
self.count_trials += 1
|
|
|
42 |
|
|
|
43 |
if self.count_trials < 3:
|
|
|
44 |
return self.read(url)
|
|
|
45 |
|
|
|
46 |
self.response_data=response_data
|
|
|
47 |
return self.createData()
|
|
|
48 |
|
|
|
49 |
def createData(self):
|
|
|
50 |
self.soup = strip_tags(self.response_data,invalid_tags)
|
|
|
51 |
self.response_data =None
|
|
|
52 |
return self.scrape(self.soup)
|
|
|
53 |
|
|
|
54 |
|
|
|
55 |
def scrape(self,soup):
|
|
|
56 |
sellerData = soup.findAll("div" , {"class" : "a-row a-spacing-mini olpOffer"})
|
|
|
57 |
for data in sellerData:
|
|
|
58 |
print "sellerData****"
|
|
|
59 |
price = data.find('span', attrs={'class' : re.compile('.*olpOfferPrice*')}).find('span').text
|
|
|
60 |
print "Unit cost= ",float(price.replace("Rs.","").replace(",",""))
|
|
|
61 |
unitCost = float(price.replace("Rs.","").replace(",",""))
|
|
|
62 |
shippingCost = data.find('p', attrs={'class' : re.compile('.*olpShippingInfo*')}).find('span').text
|
|
|
63 |
if "FREE" in shippingCost:
|
|
|
64 |
print "shippingCost=0"
|
|
|
65 |
shippingCost = 0
|
|
|
66 |
else:
|
|
|
67 |
try:
|
|
|
68 |
print "shippingCost= ",float(shippingCost.replace("+Rs.","").replace("Delivery","").replace(",",""))
|
|
|
69 |
shippingCost = float(shippingCost.replace("+Rs.","").replace("Delivery","").replace(",",""))
|
|
|
70 |
except:
|
|
|
71 |
shippingCost = 0.0
|
|
|
72 |
try:
|
|
|
73 |
return (unitCost + shippingCost)
|
|
|
74 |
except:
|
|
|
75 |
return unitCost
|
|
|
76 |
|
|
|
77 |
|
|
|
78 |
if __name__ == '__main__':
|
|
|
79 |
scraper = AmazonScraper()
|
|
|
80 |
print scraper.read('http://www.amazon.in/gp/offer-listing/B00J60MFTO/ref=olp_sort_ps')
|
|
|
81 |
|