Subversion Repositories SmartDukaan

Rev

Rev 14366 | Rev 14543 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14366 Rev 14538
Line 20... Line 20...
20
20
21 
    def read(self, url):
 21 
    def read(self, url):
22 
        response_data = ""
 22 
        response_data = ""
23 
        redirect_url = ""
 23 
        redirect_url = ""
24 
        try:
 24 
        try:
- 
 
 25
- 
 
 26 
            """quick fix,need to add it conf"""
- 
 
 27
- 
 
 28 
            proxy = urllib2.ProxyHandler({'http': 'http://192.161.163.60:8800'})
- 
 
 29 
            opener = urllib2.build_opener(proxy)
25 
            req = urllib2.Request(url,headers=headers)
 30 
            urllib2.install_opener(opener)
26 
            response = urllib2.urlopen(req)
 31 
            response = urllib2.urlopen(url)
27 
            response_data = response.read()
 32 
            response_data = response.read()
- 
 
 33 
            print response_data
28 
            response.close()
 34 
            response.close()
- 
 
 35
- 
 
 36 
#            req = urllib2.Request(url,headers=headers)
- 
 
 37 
#            response = urllib2.urlopen(req)
- 
 
 38 
            #response_data = response.read()
- 
 
 39 
            #response.close()
29 
            print "Fetched response from flipkart for %s" %(url)
 40 
            print "Fetched response from flipkart for %s" %(url)
30 
            redirect_url = response.url
 41 
            #redirect_url = response.url
31 
 
 42 
 
32 
        except Exception as e:
 43 
        except Exception as e:
33 
            print 'ERROR: ', e
 44 
            print 'ERROR: ', e
34 
            print 'Retrying'
 45 
            print 'Retrying'
35 
            self.count_trials += 1
 46 
            self.count_trials += 1
Line 75... Line 86...
75 
            except:
 86 
            except:
76 
                offerText = ""
 87 
                offerText = ""
77 
            return {'lowestSp':sellingPrice,'inStock':1}
 88 
            return {'lowestSp':sellingPrice,'inStock':1}
78 
        except:
 89 
        except:
79 
            return {'lowestSp':0,'inStock':0}
 90 
            return {'lowestSp':0,'inStock':0}
80
- 
 
81
- 
 
82
- 
 
83
- 
 
84 
 
 91 
 
85 
if __name__ == '__main__':
 92 
if __name__ == '__main__':
86 
    scraper = FlipkartProductPageScraper()
 93 
    scraper = FlipkartProductPageScraper()
87 
    print scraper.read('http://www.flipkart.com/samsung-galaxy-e7/p/itme3un6wdygtqtr?pid=MOBE3UN6NYKGPZPM')
 94 
    print scraper.read('http://www.flipkart.com/spice-stellar-glide-mi-438/p/itme5kw5jx6jgskn?pid=MOBE2U6Y5WHBHMZZ')