Subversion Repositories SmartDukaan

Rev

Rev 4198 | Rev 5291 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
4039 varun.gupt 1 
'''
2 
Created on 24-Aug-2011
3 
 
4 
@author: Varun Gupta
5 
'''
6 
 
7 
from BeautifulSoup import BeautifulSoup
8 
from BaseScraper import BaseScraper
4198 varun.gupt 9 
from Utils import removePriceFormatting
4039 varun.gupt 10 
 
11 
class FlipcartScraper(BaseScraper):
12 
 
13 
    def __init__(self):
14 
        BaseScraper.__init__(self)
15 
        self.url = None
16 
        self.id = None
17 
 
18 
    def setUrl(self, url):
19 
        self.url = url
20 
 
21 
    def scrape(self):
22 
        html = BaseScraper.read(self, self.url)
23 
        self.soup = BeautifulSoup(html)
24 
        self.phones = None
25 
 
26 
    def getPhones(self):
27 
        phones = []
28 
 
29 
        for div in self.soup.findAll('div', {'class': 'fk-product-thumb fkp-medium'}):
30 
            try:
31 
                anchor = div.findAll('a', {'class': 'title fk-anchor-link'})[0]
32 
                name = anchor['title'].strip()
33 
                price = None
34 
                product_url = anchor['href'].strip()
35 
                in_stock = 0 if div.findAll('b').__len__() > 0 else 1
36 
 
37 
                for span in div.findAll('span'):
38 
                    try:
39 
                        if span['class'].find('price final-price') > -1:
40 
                            price = span.string.strip()
41 
                    except KeyError:
42 
                        pass
43 
                try:
44 
                    if price is None:
45 
                        continue
46 
                    else:
4198 varun.gupt 47 
                        phones.append({
48 
                                'name': str(name),
49 
                                'price': removePriceFormatting(price),
50 
                                'source': 'flipkart',
51 
                                'product_url': str(product_url),
52 
                                'in_stock': in_stock
53 
                            })
4039 varun.gupt 54 
 
55 
                except UnboundLocalError as e:
56 
                    print e, name
57 
                    print div
58 
 
59 
                except UnicodeEncodeError as e:
60 
                    print 'Unicode Error', e, name
61 
                    name_ascii = "".join([char if ord(char) < 128 else " " for char in name])
62 
                    print name_ascii
4198 varun.gupt 63 
                    phones.append({
64 
                            "name": str(name_ascii),
65 
                            "price": str(price),
66 
                            'source': 'flipkart',
67 
                            "in_stock": in_stock,
68 
                            "product_url": str(product_url)
69 
                        })
4039 varun.gupt 70 
            except KeyError:
71 
                pass
72 
        self.phones = phones
73 
        return phones
74 
 
75 
    def getNextUrl(self):
4203 varun.gupt 76 
        tab_info = self.soup.find('div', {'class': 'unit fk-lres-header-text'})('b')
77 
 
78 
        current_max = int(tab_info[0].find('span').string)
4039 varun.gupt 79 
        total = int(tab_info[1].string)
80 
 
81 
        if len(self.phones) > 0:
82 
            base_url = 'http://www.flipkart.com/mobiles/%s' % ('all/' if self.phones[0]['product_url'].find('/tablets/') == -1 else 'tablet-20278/')
83 
 
84 
            if current_max < total:
85 
                return base_url + str(1 + (current_max / 20))
86 
            else:
87 
                return None
88 
        else:
89 
            return None
90 
 
4198 varun.gupt 91 
    def getDataFromProductPage(self, url):
92 
        html = BaseScraper.read(self, url)
93 
        soup = BeautifulSoup(html)
94 
        name = soup.find('h1', {'itemprop': 'name'}).string.strip()
95 
        price = soup.find('span',{'id': 'fk-mprod-our-id'}).contents[2]
96 
        in_stock = soup.find('div', {'id': 'fk-stock-info-id'}).string.strip()
97 
 
98 
        data = {
99 
            "product_url": str(url),
100 
            "source": "flipkart",
101 
            "price": price,
102 
            "in_stock": 1 if in_stock == 'In Stock.' else 0,
103 
            "name": name
104 
        }
105 
        return data
4039 varun.gupt 106 
 
107 
if __name__ == '__main__':
108 
    s = FlipcartScraper()
4203 varun.gupt 109 
#    data = s.getDataFromProductPage('http://www.flipkart.com/mobiles/micromax/itmd4nf8p5rfhk2y?pid=mobd4nf7rcrckjhn')
110 
#    print data
4198 varun.gupt 111 
 
4203 varun.gupt 112 
    s.setUrl('http://www.flipkart.com/mobiles/all/24')
113 
    s.scrape()
114 
    phones = s.getPhones()
115 
    for p in phones: print p
116 
    print s.getNextUrl()