Subversion Repositories SmartDukaan

Rev

Rev 12189 | Rev 12191 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 12189 Rev 12190
Line 13... Line 13...
13 
        request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')
 13 
        request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')
14 
        opener = urllib2.build_opener()
 14 
        opener = urllib2.build_opener()
15 
        response_data = ""
 15 
        response_data = ""
16 
        try:
 16 
        try:
17 
            response_data = opener.open(request).read()
 17 
            response_data = opener.open(request).read()
18 
            print datetime.now()
 18 
            print "Response fetched ",datetime.now()
19
19
20 
        except urllib2.HTTPError as e:
 20 
        except urllib2.HTTPError as e:
21 
            print 'ERROR: ', e
 21 
            print 'ERROR: ', e
22 
            print 'Retrying'
 22 
            print 'Retrying'
23 
            self.count_trials += 1
 23 
            self.count_trials += 1
Line 110... Line 110...
110 
                if not temp.has_key('shippingTime'):
 110 
                if not temp.has_key('shippingTime'):
111 
                    print "Populating shipping time from metrics"
 111 
                    print "Populating shipping time from metrics"
112 
                    temp['shippingTime'] = dataMetric[3]
 112 
                    temp['shippingTime'] = dataMetric[3]
113 
                temp['sellerScore'] = int(dataMetric[4])
 113 
                temp['sellerScore'] = int(dataMetric[4])
114 
                info.append(temp)
 114 
                info.append(temp)
- 
 
 115 
        print "Returning response ",datetime.now()
115 
        return info
 116 
        return info
116 
 
 117 
 
117 
if __name__ == '__main__':
 118 
if __name__ == '__main__':
118 
    print datetime.now()
 119 
    print datetime.now()
119 
    scraper = FlipkartScraper()
 120 
    scraper = FlipkartScraper()