Subversion Repositories SmartDukaan

Rev

Rev 15155 | Rev 15215 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 15155 Rev 15212
Line 21... Line 21...
21 
            tag.replaceWith(s)
 21 
            tag.replaceWith(s)
22 
 
 22 
 
23 
    return soup
 23 
    return soup
24 
 
 24 
 
25 
class AmazonScraper:
 25 
class AmazonScraper:
26 
    def __init__(self):
 26 
    def __init__(self, livePricing=None):
27 
        self.count_trials = 0
 27 
        self.count_trials = 0
- 
 
 28 
        self.livePricing = livePricing
28
29
29 
    def read(self, url):
 30 
    def read(self, url):
30 
        response_data = ""
 31 
        response_data = ""
- 
 
 32 
        print self.livePricing
31 
        try:
 33 
        try:
32 
            response_data = fetchResponseUsingProxy(url)
 34 
            response_data = fetchResponseUsingProxy(url,livePricing=self.livePricing)
33 
        except Exception as e:
 35 
        except Exception as e:
34 
            print 'ERROR: ', e
 36 
            print 'ERROR: ', e
35 
            print 'Retrying'
 37 
            print 'Retrying'
36 
            self.count_trials += 1
 38 
            self.count_trials += 1
37
39
38 
            if self.count_trials < 5:
 40 
            if self.count_trials < 5:
39 
                return self.read(url)
 41 
                return self.read(url)
40
42
41 
        self.response_data=response_data
 43 
        self.response_data=response_data
- 
 
 44 
        print response_data
42 
        if "Server Busy" in self.response_data:
 45 
        if "Server Busy" in self.response_data:
43 
            print "Server busy...Ahhhhh"
 46 
            print "Server busy...Ahhhhh"
44 
            self.count_trials += 1
 47 
            self.count_trials += 1
45 
            return self.read(url)
 48 
            return self.read(url)
46 
        return self.createData()
 49 
        return self.createData()
Line 73... Line 76...
73 
            except:
 76 
            except:
74 
                return  unitCost
 77 
                return  unitCost
75
78
76 
 
 79 
 
77 
if __name__ == '__main__':
 80 
if __name__ == '__main__':
78 
    scraper = AmazonScraper()
 81 
    scraper = AmazonScraper(True)
79 
    print scraper.read('http://www.amazon.in/gp/offer-listing/B00UFPHX8M')
 82 
    print scraper.read('http://www.amazon.in/gp/offer-listing/B00UFPHX8M')
80
83
81 

            84