| Line 1... |
Line -... |
| 1 |
from BeautifulSoup import BeautifulSoup
|
- |
|
| 2 |
from dtr.utils.utils import fetchResponseUsingProxy
|
1 |
from dtr.utils.utils import fetchResponseUsingProxy, transformUrl
|
| 3 |
from sys import exit
|
2 |
from sys import exit
|
| 4 |
import json
|
3 |
import json
|
| 5 |
import re
|
4 |
import re
|
| 6 |
import traceback
|
5 |
import traceback
|
| - |
|
6 |
import datetime
|
| - |
|
7 |
from pyquery import PyQuery
|
| - |
|
8 |
|
| 7 |
|
9 |
|
| 8 |
|
10 |
|
| 9 |
headers = {
|
11 |
headers = {
|
| 10 |
'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',
|
12 |
'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',
|
| 11 |
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
13 |
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
| 12 |
'Accept-Language' : 'en-US,en;q=0.8',
|
14 |
'Accept-Language' : 'en-US,en;q=0.8',
|
| 13 |
'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
|
15 |
'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
|
| 14 |
'Cookie':'T=TI141257426738726661427143281839817329423126740566618323641725716448; __sonar=7237334677420142002; __gads=ID=c8b82101a0e4f451:T=1412574724:S=ALNI_MbPMbEOZj2nAGjM54z8ZHFMqwTOTQ; FK-CMP-DATA=; SN=2.VI11FB3FB6ED9D4693A796AB8C965B3417.SI802C325AC43444858830E870C4FD3324.VS141257426735693951472.1412576209; VID=2.VI11FB3FB6ED9D4693A796AB8C965B3417.1412576209.VS141257426735693951472; NSID=2.SI802C325AC43444858830E870C4FD3324.1412576209.VI11FB3FB6ED9D4693A796AB8C965B3417; __utma=19769839.709301254.1412574234.1412574234.1412574234.1; __utmb=19769839.23.10.1412574234; __utmc=19769839; __utmz=19769839.1412574234.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); s_cc=true; gpv_pn=SellerListing%3AMobile%3AMicromax%20Canvas%20Fire%20A093; gpv_pn_t=no%20value; s_sq=%5B%5BB%5D%5D; pincode=110011; s_ppv=36',
|
- |
|
| 15 |
'Connection':'keep-alive',
|
16 |
'Connection':'keep-alive',
|
| 16 |
'Accept-Encoding' : 'gzip,deflate,sdch'
|
17 |
'Accept-Encoding' : 'gzip,deflate,sdch'
|
| 17 |
}
|
18 |
}
|
| 18 |
|
19 |
|
| - |
|
20 |
|
| 19 |
class FlipkartProductPageScraper:
|
21 |
class FlipkartProductPageScraper:
|
| 20 |
def __init__(self):
|
22 |
def __init__(self):
|
| 21 |
self.count_trials = 0
|
23 |
self.count_trials = 0
|
| 22 |
self.redirectCount = 0
|
24 |
self.redirectCount = 0
|
| 23 |
|
25 |
|
| 24 |
def read(self, url):
|
26 |
def read(self, url):
|
| 25 |
response_data = ""
|
27 |
response_data = ""
|
| 26 |
redirect_url = ""
|
28 |
redirect_url = ""
|
| - |
|
29 |
url = transformUrl(url,2)
|
| 27 |
try:
|
30 |
try:
|
| 28 |
|
31 |
|
| 29 |
"""quick fix,need to add it conf"""
|
32 |
"""quick fix,need to add it conf"""
|
| 30 |
|
33 |
|
| 31 |
response_data = fetchResponseUsingProxy(url, headers)
|
34 |
response_data = fetchResponseUsingProxy(url, headers, proxy=True)
|
| 32 |
print "Fetched response from flipkart for %s" %(url)
|
35 |
print "Fetched response from flipkart for %s" %(url)
|
| 33 |
#redirect_url = response.url
|
36 |
#redirect_url = response.url
|
| 34 |
|
37 |
|
| 35 |
except Exception as e:
|
38 |
except Exception as e:
|
| 36 |
traceback.print_exc()
|
39 |
traceback.print_exc()
|
| Line 40... |
Line 43... |
| 40 |
|
43 |
|
| 41 |
if self.count_trials < 3:
|
44 |
if self.count_trials < 3:
|
| 42 |
return self.read(url)
|
45 |
return self.read(url)
|
| 43 |
|
46 |
|
| 44 |
self.response_data=response_data
|
47 |
self.response_data=response_data
|
| - |
|
48 |
print datetime.datetime.now()
|
| 45 |
return self.createData(url,redirect_url)
|
49 |
return self.createData(url,redirect_url)
|
| 46 |
|
50 |
|
| 47 |
def scrapeRedirectedPage(self,soup,redirect_url):
|
- |
|
| 48 |
print soup
|
- |
|
| 49 |
print redirect_url
|
- |
|
| 50 |
t = soup.find("div" , {"class" : "seller-table fk-user-select-none line"})
|
- |
|
| 51 |
print t
|
- |
|
| 52 |
table_rows = t.findAll("tr" , {"class" : re.compile('t-row.*')})
|
- |
|
| 53 |
print table_rows
|
- |
|
| 54 |
for x in table_rows:
|
- |
|
| 55 |
print x
|
- |
|
| 56 |
|
- |
|
| 57 |
def createData(self,url, redirect_url):
|
51 |
def createData(self,url, redirect_url):
|
| 58 |
print "Creating soup from flipkart data for %s" %(url)
|
- |
|
| 59 |
#redirect_url = redirect_url.replace('www.flipkart.com','163.53.77.21')
|
- |
|
| 60 |
print "Redirect url is %s"%(redirect_url)
|
- |
|
| 61 |
page=self.response_data.decode("utf-8")
|
52 |
pq = PyQuery(self.response_data)
|
| 62 |
self.soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
|
53 |
buyBoxPrice = float(pq('span.selling-price.omniture-field').attr['data-evar48'])
|
| 63 |
page = None
|
54 |
inStock = 1
|
| 64 |
self.response_data = None
|
- |
|
| 65 |
print "Soup created from flipkart data for %s" %(url)
|
- |
|
| 66 |
print redirect_url
|
- |
|
| 67 |
return self.scrape(self.soup,url)
|
- |
|
| 68 |
|
- |
|
| 69 |
def scrape(self,soup,url):
|
- |
|
| 70 |
try:
|
55 |
try:
|
| 71 |
print "data-config"
|
- |
|
| 72 |
if soup.find('div',{'class':'seller-table-wrap section'}) is None:
|
56 |
sellerJson = pq('div.seller-table-wrap').attr['data-config']
|
| 73 |
raise
|
57 |
x = json.loads(sellerJson)
|
| 74 |
x = json.loads(soup.find('div',{'class':'seller-table-wrap section'})['data-config'])['dataModel']
|
- |
|
| 75 |
lines = sorted(x, key=lambda k: k['priceInfo'].get('sellingPrice', 0), reverse=False)
|
58 |
lines = sorted(x['dataModel'], key=lambda k: k['priceInfo'].get('sellingPrice', 0), reverse=False)
|
| 76 |
sellingPrice = float(lines[0]['priceInfo']['sellingPrice'])
|
59 |
sellingPrice = float(lines[0]['priceInfo']['sellingPrice'])
|
| 77 |
try:
|
60 |
try:
|
| 78 |
offerText = lines[0]['offerInfo']['listingOffers'][0]['description']
|
61 |
offerText = lines[0]['offerInfo']['listingOffers'][0]['description']
|
| 79 |
except:
|
62 |
except:
|
| 80 |
offerText = ""
|
63 |
offerText = ""
|
| 81 |
buyBoxPrice = float(soup.find('span',{'class':'selling-price omniture-field'})['data-evar48'])
|
- |
|
| 82 |
return {'lowestSp':sellingPrice,'inStock':1,'buyBoxPrice':buyBoxPrice}
|
64 |
return {'lowestSp':sellingPrice,'inStock':inStock,'buyBoxPrice':buyBoxPrice}
|
| 83 |
except:
|
65 |
except:
|
| 84 |
"""No able to parse seller wrap section, probably due to only single seller option"""
|
66 |
"""Not able to parse seller wrap section, probably due to only single seller option"""
|
| 85 |
buyBoxPrice = float(soup.find('span',{'class':'selling-price omniture-field'})['data-evar48'])
|
- |
|
| 86 |
sellingPrice = buyBoxPrice
|
67 |
sellingPrice = buyBoxPrice
|
| 87 |
if soup.find('div',{'class':'out-of-stock'}) is not None:
|
68 |
stockDiv = pq('div.out-of-stock')
|
| - |
|
69 |
if len(stockDiv) > 0:
|
| 88 |
inStock = 0
|
70 |
inStock = 0
|
| 89 |
else:
|
- |
|
| 90 |
inStock = 1
|
- |
|
| 91 |
return {'lowestSp':sellingPrice,'inStock':inStock,'buyBoxPrice':buyBoxPrice}
|
71 |
return {'lowestSp':sellingPrice,'inStock':inStock,'buyBoxPrice':buyBoxPrice}
|
| 92 |
|
72 |
|
| 93 |
if __name__ == '__main__':
|
73 |
if __name__ == '__main__':
|
| - |
|
74 |
print datetime.datetime.now()
|
| 94 |
scraper = FlipkartProductPageScraper()
|
75 |
scraper = FlipkartProductPageScraper()
|
| - |
|
76 |
|
| 95 |
print scraper.read('http://www.flipkart.com/samsung-galaxy-star-advance/p/itmeyfc4vwzwhuva?pid=MOBEYFC44EUDZ9TX')
|
77 |
print scraper.read('http://www.flipkart.com/redmi-2/p/itme8ygtcfax6w39')
|
| - |
|
78 |
print datetime.datetime.now()
|