| 16408 |
kshitij.so |
1 |
from dtr.utils.utils import fetchResponseUsingProxy, transformUrl
|
|
|
2 |
from sys import exit
|
|
|
3 |
import json
|
|
|
4 |
import traceback
|
|
|
5 |
import datetime
|
|
|
6 |
import chardet
|
|
|
7 |
|
|
|
8 |
|
|
|
9 |
class PaytmScraper:
|
|
|
10 |
def __init__(self):
|
|
|
11 |
self.count_trials = 0
|
|
|
12 |
self.redirectCount = 0
|
|
|
13 |
|
|
|
14 |
def read(self, url):
|
|
|
15 |
response_data = ""
|
|
|
16 |
try:
|
|
|
17 |
|
|
|
18 |
"""quick fix,need to add it conf"""
|
|
|
19 |
|
|
|
20 |
response_data = fetchResponseUsingProxy(url, proxy=True)
|
|
|
21 |
print "Fetched response from paytm for %s" %(url)
|
|
|
22 |
#redirect_url = response.url
|
|
|
23 |
|
|
|
24 |
except Exception as e:
|
|
|
25 |
traceback.print_exc()
|
|
|
26 |
print 'ERROR: ', e
|
|
|
27 |
print 'Retrying'
|
|
|
28 |
self.count_trials += 1
|
|
|
29 |
|
|
|
30 |
if self.count_trials < 3:
|
|
|
31 |
return self.read(url)
|
|
|
32 |
|
|
|
33 |
self.response_data=response_data
|
|
|
34 |
return self.createData()
|
|
|
35 |
|
|
|
36 |
def createData(self):
|
|
|
37 |
try:
|
|
|
38 |
encoding = chardet.detect(self.response_data)
|
|
|
39 |
try:
|
|
|
40 |
self.response_data = self.response_data.decode(encoding.get('encoding'))
|
|
|
41 |
except:
|
|
|
42 |
self.response_data = self.response_data.decode('ascii')
|
|
|
43 |
input_json = json.loads(self.response_data)
|
|
|
44 |
offerPrice = float(input_json['offer_price'])
|
|
|
45 |
cod = int(input_json['pay_type_supported'].get('COD'))
|
|
|
46 |
offerUrl = (input_json['offer_url'])
|
|
|
47 |
inStock = (input_json['instock'])
|
|
|
48 |
return {'offerPrice':offerPrice,'codAvailable':cod,'offerUrl':offerUrl,'inStock':inStock}
|
|
|
49 |
except:
|
|
|
50 |
return {'offerPrice':0.0,'codAvailable':0,'offerUrl':"",'inStock':False}
|
|
|
51 |
|
|
|
52 |
if __name__ == '__main__':
|
|
|
53 |
scraper = PaytmScraper()
|
|
|
54 |
print scraper.read('https://catalog.paytm.com/v1/mobile/product/1849504')
|