Subversion Repositories SmartDukaan

Rev

Rev 4039 | Rev 4203 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
4039 varun.gupt 1 
'''
2 
Created on 24-Aug-2011
3 
 
4 
@author: Varun Gupta
5 
'''
6 
 
7 
from BeautifulSoup import BeautifulSoup
8 
from BaseScraper import BaseScraper
4198 varun.gupt 9 
from Utils import removePriceFormatting
4039 varun.gupt 10 
 
11 
class FlipcartScraper(BaseScraper):
12 
 
13 
    def __init__(self):
14 
        BaseScraper.__init__(self)
15 
        self.url = None
16 
        self.id = None
17 
 
18 
    def setUrl(self, url):
19 
        self.url = url
20 
 
21 
    def scrape(self):
22 
        html = BaseScraper.read(self, self.url)
23 
        self.soup = BeautifulSoup(html)
24 
        self.phones = None
25 
 
26 
    def getPhones(self):
27 
        phones = []
28 
 
29 
        for div in self.soup.findAll('div', {'class': 'fk-product-thumb fkp-medium'}):
30 
            try:
31 
                anchor = div.findAll('a', {'class': 'title fk-anchor-link'})[0]
32 
                name = anchor['title'].strip()
33 
                price = None
34 
                product_url = anchor['href'].strip()
35 
                in_stock = 0 if div.findAll('b').__len__() > 0 else 1
36 
 
37 
                for span in div.findAll('span'):
38 
                    try:
39 
                        if span['class'].find('price final-price') > -1:
40 
                            price = span.string.strip()
41 
                    except KeyError:
42 
                        pass
43 
                try:
44 
                    if price is None:
45 
                        continue
46 
                    else:
4198 varun.gupt 47 
                        phones.append({
48 
                                'name': str(name),
49 
                                'price': removePriceFormatting(price),
50 
                                'source': 'flipkart',
51 
                                'product_url': str(product_url),
52 
                                'in_stock': in_stock
53 
                            })
4039 varun.gupt 54 
 
55 
                except UnboundLocalError as e:
56 
                    print e, name
57 
                    print div
58 
 
59 
                except UnicodeEncodeError as e:
60 
                    print 'Unicode Error', e, name
61 
                    name_ascii = "".join([char if ord(char) < 128 else " " for char in name])
62 
                    print name_ascii
4198 varun.gupt 63 
                    phones.append({
64 
                            "name": str(name_ascii),
65 
                            "price": str(price),
66 
                            'source': 'flipkart',
67 
                            "in_stock": in_stock,
68 
                            "product_url": str(product_url)
69 
                        })
4039 varun.gupt 70 
            except KeyError:
71 
                pass
72 
        self.phones = phones
73 
        return phones
74 
 
75 
    def getNextUrl(self):
76 
        tab_info = self.soup.findAll('div', {'class': 'unit fk-lres-header-text'})[0]('b')
77 
        current_max = int(tab_info[0].string.split('-')[1])
78 
        total = int(tab_info[1].string)
79 
 
80 
        if len(self.phones) > 0:
81 
            base_url = 'http://www.flipkart.com/mobiles/%s' % ('all/' if self.phones[0]['product_url'].find('/tablets/') == -1 else 'tablet-20278/')
82 
 
83 
            if current_max < total:
84 
                return base_url + str(1 + (current_max / 20))
85 
            else:
86 
                return None
87 
        else:
88 
            return None
89 
 
4198 varun.gupt 90 
    def getDataFromProductPage(self, url):
91 
        html = BaseScraper.read(self, url)
92 
        soup = BeautifulSoup(html)
93 
        name = soup.find('h1', {'itemprop': 'name'}).string.strip()
94 
        price = soup.find('span',{'id': 'fk-mprod-our-id'}).contents[2]
95 
        in_stock = soup.find('div', {'id': 'fk-stock-info-id'}).string.strip()
96 
 
97 
        data = {
98 
            "product_url": str(url),
99 
            "source": "flipkart",
100 
            "price": price,
101 
            "in_stock": 1 if in_stock == 'In Stock.' else 0,
102 
            "name": name
103 
        }
104 
        return data
4039 varun.gupt 105 
 
106 
if __name__ == '__main__':
107 
    s = FlipcartScraper()
4198 varun.gupt 108 
    data = s.getDataFromProductPage('http://www.flipkart.com/mobiles/micromax/itmd4nf8p5rfhk2y?pid=mobd4nf7rcrckjhn')
109 
    print data
110 
 
111 
#    s.setUrl('http://www.flipkart.com/mobiles/all/27')
112 
#    s.scrape()
113 
#    phones = s.getPhones()
114 
#    for p in phones: print p
115 
#    print s.getNextUrl()