Subversion Repositories SmartDukaan

Rev

Rev 11967 | Rev 12190 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 11967 Rev 12189
Line 1... Line 1...
1 
import urllib2
 1 
import urllib2
2 
from BeautifulSoup import BeautifulSoup
 2 
from BeautifulSoup import BeautifulSoup
3 
import re
 3 
import re
4 
from sys import exit
 4 
from sys import exit
- 
 
 5 
from datetime import datetime
5 
 
 6 
 
6 
class FlipkartScraper:
 7 
class FlipkartScraper:
7 
    def __init__(self):
 8 
    def __init__(self):
8 
        self.count_trials = 0
 9 
        self.count_trials = 0
9
10
Line 12... Line 13...
12 
        request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')
 13 
        request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')
13 
        opener = urllib2.build_opener()
 14 
        opener = urllib2.build_opener()
14 
        response_data = ""
 15 
        response_data = ""
15 
        try:
 16 
        try:
16 
            response_data = opener.open(request).read()
 17 
            response_data = opener.open(request).read()
- 
 
 18 
            print datetime.now()
17
19
18 
        except urllib2.HTTPError as e:
 20 
        except urllib2.HTTPError as e:
19 
            print 'ERROR: ', e
 21 
            print 'ERROR: ', e
20 
            print 'Retrying'
 22 
            print 'Retrying'
21 
            self.count_trials += 1
 23 
            self.count_trials += 1
Line 111... Line 113...
111 
                temp['sellerScore'] = int(dataMetric[4])
 113 
                temp['sellerScore'] = int(dataMetric[4])
112 
                info.append(temp)
 114 
                info.append(temp)
113 
        return info
 115 
        return info
114 
 
 116 
 
115 
if __name__ == '__main__':
 117 
if __name__ == '__main__':
- 
 
 118 
    print datetime.now()
116 
    scraper = FlipkartScraper()
 119 
    scraper = FlipkartScraper()
117 
    scraper.read('http://www.flipkart.com/ps/ACCDSN84XXH5P9WG')
 120 
    scraper.read('http://www.flipkart.com/ps/ACCDSN84XXH5P9WG')
118 
    print scraper.createData()
 121 
    print scraper.createData()
- 
 
 122 
    print datetime.now()
119
123
120 

            124