Subversion Repositories SmartDukaan

Rev

Rev 11217 | Rev 11967 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 11217 Rev 11668
Line 1... Line 1...
1 
import urllib2
 1 
import urllib2
2 
from BeautifulSoup import BeautifulSoup
 2 
from BeautifulSoup import BeautifulSoup
3 
import re
 3 
import re
- 
 
 4 
from sys import exit
4 
 
 5 
 
5 
class FlipkartScraper:
 6 
class FlipkartScraper:
6 
    def __init__(self):
 7 
    def __init__(self):
7 
        self.count_trials = 0
 8 
        self.count_trials = 0
8
9
Line 33... Line 34...
33 
    def scrape(self,soup):
 34 
    def scrape(self,soup):
34 
        info = []
 35 
        info = []
35 
        oddSeller = soup.findAll("div" , {"class" : "line seller-item odd "})
 36 
        oddSeller = soup.findAll("div" , {"class" : "line seller-item odd "})
36 
        for data in oddSeller:
 37 
        for data in oddSeller:
37 
            temp={}
 38 
            temp={}
- 
 
 39 
            try:
- 
 
 40 
                businessDays = data.find('span', attrs={'class' : re.compile('fk-deliverable.*')})
- 
 
 41 
                shippingTime = businessDays.find('span', attrs={'class' : re.compile('fk-bold')}).string.replace('to','').replace('business days.','').strip().replace('  ','-')
- 
 
 42 
                temp['shippingTime']=shippingTime
- 
 
 43 
            except:
- 
 
 44 
                pass
38 
            price = data.find('span', attrs={'class' : re.compile('pxs-final-price.*')}).string.strip('Rs.').strip()
 45 
            price = data.find('span', attrs={'class' : re.compile('pxs-final-price.*')}).string.strip('Rs.').strip()
39 
            temp['sellingPrice']=float(price)
 46 
            temp['sellingPrice']=float(price)
40 
            for sellerInfo in data.findAll("div",{"class":re.compile(".*seller-info*")}):
 47 
            for sellerInfo in data.findAll("div",{"class":re.compile(".*seller-info*")}):
41 
                sellerName = sellerInfo.find('a').string
 48 
                sellerName = sellerInfo.find('a').string
42 
                temp['sellerName'] = sellerName
 49 
                temp['sellerName'] = sellerName
Line 57... Line 64...
57 
                temp['buyTrend']=buyTrend
 64 
                temp['buyTrend']=buyTrend
58 
                dataMetric = dataMetrics.split(';')
 65 
                dataMetric = dataMetrics.split(';')
59 
                sellerCode = dataMetric[0]
 66 
                sellerCode = dataMetric[0]
60 
                temp['sellerCode']=sellerCode
 67 
                temp['sellerCode']=sellerCode
61 
                temp['sellingPriceMetric'] = float(dataMetric[1])
 68 
                temp['sellingPriceMetric'] = float(dataMetric[1])
- 
 
 69 
                if not temp.has_key('shippingTime'):
- 
 
 70 
                    print "Populating shipping time from metrics"
62 
                temp['shippingTime'] = dataMetric[3]
 71 
                    temp['shippingTime'] = dataMetric[3]
63 
                temp['sellerScore'] = int(dataMetric[4])
 72 
                temp['sellerScore'] = int(dataMetric[4])
64 
                info.append(temp)
 73 
                info.append(temp)
65 
        evenSeller = soup.findAll("div" , {"class" : "line seller-item even "})
 74 
        evenSeller = soup.findAll("div" , {"class" : "line seller-item even "})
66 
        for data in evenSeller:
 75 
        for data in evenSeller:
67 
            temp={}
 76 
            temp={}
68 
            price = data.find('span', attrs={'class' : re.compile('pxs-final-price.*')}).string.strip('Rs.')
 77 
            price = data.find('span', attrs={'class' : re.compile('pxs-final-price.*')}).string.strip('Rs.')
- 
 
 78 
            try:
- 
 
 79 
                businessDays = data.find('span', attrs={'class' : re.compile('fk-deliverable.*')})
- 
 
 80 
                shippingTime = businessDays.find('span', attrs={'class' : re.compile('fk-bold')}).string.replace('to','').replace('business days.','').strip().replace('  ','-')
- 
 
 81 
                temp['shippingTime']=shippingTime
- 
 
 82 
            except:
- 
 
 83 
                pass
69 
            temp['sellingPrice']=float(price)
 84 
            temp['sellingPrice']=float(price)
70 
            for sellerInfo in data.findAll("div",{"class":re.compile(".*seller-info*")}):
 85 
            for sellerInfo in data.findAll("div",{"class":re.compile(".*seller-info*")}):
71 
                sellerName = sellerInfo.find('a').string
 86 
                sellerName = sellerInfo.find('a').string
72 
                temp['sellerName'] = sellerName
 87 
                temp['sellerName'] = sellerName
73 
            for metrics in data.find("div",{"class":"fk-text-right"}):
 88 
            for metrics in data.find("div",{"class":"fk-text-right"}):
Line 86... Line 101...
86 
                    buyTrend = inputTags[0:str(inputTags).index('WSR')].replace('_','')
 101 
                    buyTrend = inputTags[0:str(inputTags).index('WSR')].replace('_','')
87 
                temp['buyTrend']=buyTrend
 102 
                temp['buyTrend']=buyTrend
88 
                dataMetric = dataMetrics.split(';')
 103 
                dataMetric = dataMetrics.split(';')
89 
                temp['sellerCode'] = dataMetric[0]
104 
                temp['sellerCode'] = dataMetric[0]
90 
                temp['sellingPriceMetric'] = float(dataMetric[1])
 105 
                temp['sellingPriceMetric'] = float(dataMetric[1])
- 
 
 106 
                if not temp.has_key('shippingTime'):
- 
 
 107 
                    print "Populating shipping time from metrics"
91 
                temp['shippingTime'] = dataMetric[3]
 108 
                    temp['shippingTime'] = dataMetric[3]
92 
                temp['sellerScore'] = int(dataMetric[4])
 109 
                temp['sellerScore'] = int(dataMetric[4])
93 
                info.append(temp)
 110 
                info.append(temp)
94 
        return info
 111 
        return info
95 
 
 112 
 
96 
if __name__ == '__main__':
 113 
if __name__ == '__main__':
97 
    scraper = FlipkartScraper()
 114 
    scraper = FlipkartScraper()
98 
    scraper.read('http://www.flipkart.com/ps/ACCCZJZNQYSRGH8F')
 115 
    scraper.read('http://www.flipkart.com/ps/ACCDNKARGGKYVGZZ')
99 
    print scraper.createData()
 116 
    print scraper.createData()
100
117
101 

            118