Subversion Repositories SmartDukaan

Rev

Rev 12199 | Rev 12202 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 12199 Rev 12200
Line 24... Line 24...
24 
            if self.count_trials < 3:
 24 
            if self.count_trials < 3:
25 
                return self.read(url)
 25 
                return self.read(url)
26
26
27 
        self.response_data=response_data
 27 
        self.response_data=response_data
28
28
29 
    def createData(self):
 29 
    def createData(self,url):
30 
        page=self.response_data.decode("utf-8")
 30 
        page=self.response_data.decode("utf-8")
31 
        self.soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
 31 
        self.soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
32 
        page = None
 32 
        page = None
33 
        self.response_data = None
 33 
        self.response_data = None
- 
 
 34 
        print "Soup created from flipkart data for %s" %(url)
34 
        return self.scrape(self.soup)
 35 
        return self.scrape(self.soup,url)
35
36
36
37
37 
    def scrape(self,soup):
 38 
    def scrape(self,soup,url):
38 
        info = []
 39 
        info = []
39 
        oddSeller = soup.findAll("div" , {"class" : "line seller-item odd "})
 40 
        oddSeller = soup.findAll("div" , {"class" : "line seller-item odd "})
40 
        for data in oddSeller:
 41 
        for data in oddSeller:
41 
            temp={}
 42 
            temp={}
42 
            try:
 43 
            try:
Line 109... Line 110...
109 
                if not temp.has_key('shippingTime'):
 110 
                if not temp.has_key('shippingTime'):
110 
                    print "Populating shipping time from metrics"
 111 
                    print "Populating shipping time from metrics"
111 
                    temp['shippingTime'] = dataMetric[3]
 112 
                    temp['shippingTime'] = dataMetric[3]
112 
                temp['sellerScore'] = int(dataMetric[4])
 113 
                temp['sellerScore'] = int(dataMetric[4])
113 
                info.append(temp)
 114 
                info.append(temp)
- 
 
 115 
        print "Returning Json response from flipkart for %s" %(url)
114 
        return info
 116 
        return info
115 
 
 117 
 
116 
if __name__ == '__main__':
 118 
if __name__ == '__main__':
117 
    scraper = FlipkartScraper()
 119 
    scraper = FlipkartScraper()
118 
    scraper.read('http://www.flipkart.com/ps/ACCDSN84XXH5P9WG')
 120 
    scraper.read('http://www.flipkart.com/ps/ACCDSN84XXH5P9WG')
119 
    print scraper.createData()
 121 
    print scraper.createData('http://www.flipkart.com/ps/ACCDSN84XXH5P9WG')
120
122
121 

            123