Subversion Repositories SmartDukaan

Rev

Rev 12256 | Rev 12765 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 12256 Rev 12764
Line 4... Line 4...
4 
from sys import exit
 4 
from sys import exit
5 
 
 5 
 
6 
class FlipkartScraper:
 6 
class FlipkartScraper:
7 
    def __init__(self):
 7 
    def __init__(self):
8 
        self.count_trials = 0
 8 
        self.count_trials = 0
- 
 
 9 
        self.redirectCount = 0
9
10
10 
    def read(self, url):
 11 
    def read(self, url):
11 
        request = urllib2.Request(url)
 12 
        request = urllib2.Request(url)
12 
        request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')
 13 
        request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686; rv:31.0) Gecko/20100101 Firefox/31.0')
13 
        opener = urllib2.build_opener()
 14 
        opener = urllib2.build_opener()
14 
        response_data = ""
 15 
        response_data = ""
15 
        try:
 16 
        try:
- 
 
 17 
            response = urllib2.urlopen(request)
16 
            response_data = opener.open(request).read()
 18 
            response_data = response.read()
17 
            print "Fetched response from flipkart for %s" %(url)
 19 
            print "Fetched response from flipkart for %s" %(url)
- 
 
 20 
            redirect_url = response.url
18
21
19 
        except urllib2.HTTPError as e:
 22 
        except urllib2.HTTPError as e:
20 
            print 'ERROR: ', e
 23 
            print 'ERROR: ', e
21 
            print 'Retrying'
 24 
            print 'Retrying'
22 
            self.count_trials += 1
 25 
            self.count_trials += 1
23
26
24 
            if self.count_trials < 3:
 27 
            if self.count_trials < 3:
25 
                return self.read(url)
 28 
                return self.read(url)
26
29
27 
        self.response_data=response_data
 30 
        self.response_data=response_data
28 
        return self.createData(url)
 31 
        return self.createData(url,redirect_url)
29
32
- 
 
 33 
    def scrapeRedirectedPage(self,soup,redirect_url):
- 
 
 34 
        print soup
- 
 
 35 
        print redirect_url
- 
 
 36 
        t = soup.find("div" , {"class" : "seller-table fk-user-select-none line"})
- 
 
 37 
        print t
- 
 
 38 
        table_rows = t.findAll("tr" , {"class" : re.compile('t-row.*')})
- 
 
 39 
        print table_rows
- 
 
 40 
        for x in table_rows:
- 
 
 41 
            print x
- 
 
 42
30 
    def createData(self,url):
 43 
    def createData(self,url, redirect_url):
31 
        print "Creating soup from flipkart data for %s" %(url)
 44 
        print "Creating soup from flipkart data for %s" %(url)
- 
 
 45 
        print redirect_url
32 
        page=self.response_data.decode("utf-8")
 46 
        page=self.response_data.decode("utf-8")
33 
        self.soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
 47 
        self.soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
34 
        page = None
 48 
        page = None
35 
        self.response_data = None
 49 
        self.response_data = None
36 
        print "Soup created from flipkart data for %s" %(url)
 50 
        print "Soup created from flipkart data for %s" %(url)
- 
 
 51 
        if (url==redirect_url):
37 
        return self.scrape(self.soup,url)
 52 
            return self.scrape(self.soup,url)
- 
 
 53 
        else:
- 
 
 54 
            print self.redirectCount
- 
 
 55 
            self.redirectCount+=1
- 
 
 56 
            if self.redirectCount >4:
- 
 
 57 
                raise
- 
 
 58 
            return self.read(url)
- 
 
 59
- 
 
 60
38
61
39
62
40 
    def scrape(self,soup,url):
 63 
    def scrape(self,soup,url):
41 
        print "Inside json creator for %s" %(url)
 64 
        print "Inside json creator for %s" %(url)
42 
        info = []
 65 
        info = []
Line 118... Line 141...
118 
        print "Returning Json response from flipkart for %s" %(url)
 141 
        print "Returning Json response from flipkart for %s" %(url)
119 
        return info
 142 
        return info
120 
 
 143 
 
121 
if __name__ == '__main__':
 144 
if __name__ == '__main__':
122 
    scraper = FlipkartScraper()
 145 
    scraper = FlipkartScraper()
123 
    print scraper.read('http://www.flipkart.com/ps/MOBDJ4ZNAPHAUMSH')
 - 
 
124
- 
 
125 

            146
            
    print scraper.read('http://www.flipkart.com/ps/MOBDYFURT9PKAPSX')

          

          

            -
            
 

            147