Subversion Repositories SmartDukaan

Rev

Rev 14746 | Rev 15950 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14746 Rev 15265
Line 1... Line 1...
1 
from BeautifulSoup import BeautifulSoup
 1 
from BeautifulSoup import BeautifulSoup
2 
from dtr.utils.utils import fetchResponseUsingProxy
 2 
from dtr.utils.utils import fetchResponseUsingProxy
3 
from sys import exit
 3 
from sys import exit
4 
import json
 4 
import json
5 
import re
 5 
import re
- 
 
 6 
import traceback
6 
 
 7 
 
7 
 
 8 
 
8 
headers = {
9 
headers = {
9 
            'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',
 10 
            'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',
10 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
11 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
Line 30... Line 31...
30 
            response_data = fetchResponseUsingProxy(url, headers)
31 
            response_data = fetchResponseUsingProxy(url, headers)
31 
            print "Fetched response from flipkart for %s" %(url)
 32 
            print "Fetched response from flipkart for %s" %(url)
32 
            #redirect_url = response.url
 33 
            #redirect_url = response.url
33 
 
 34 
 
34 
        except Exception as e:
 35 
        except Exception as e:
35 
            import traceback
 - 
 
36 
            traceback.print_exc()
 36 
            traceback.print_exc()
37 
            print 'ERROR: ', e
 37 
            print 'ERROR: ', e
38 
            print 'Retrying'
 38 
            print 'Retrying'
39 
            self.count_trials += 1
 39 
            self.count_trials += 1
40 
 
 40 
 
Line 76... Line 76...
76 
            sellingPrice =  float(lines[0]['priceInfo']['sellingPrice'])
 76 
            sellingPrice =  float(lines[0]['priceInfo']['sellingPrice'])
77 
            try:
 77 
            try:
78 
                offerText = lines[0]['offerInfo']['listingOffers'][0]['description']
 78 
                offerText = lines[0]['offerInfo']['listingOffers'][0]['description']
79 
            except:
 79 
            except:
80 
                offerText = ""
 80 
                offerText = ""
- 
 
 81 
            buyBoxPrice = float(soup.find('span',{'class':'selling-price omniture-field'})['data-evar48'])
81 
            return {'lowestSp':sellingPrice,'inStock':1}
 82 
            return {'lowestSp':sellingPrice,'inStock':1,'buyBoxPrice':buyBoxPrice}
82 
        except:
 83 
        except:
- 
 
 84 
            """No able to parse seller wrap section, probably due to only single seller option"""
- 
 
 85 
            buyBoxPrice = float(soup.find('span',{'class':'selling-price omniture-field'})['data-evar48'])
- 
 
 86 
            sellingPrice = buyBoxPrice
- 
 
 87 
            if soup.find('div',{'class':'out-of-stock'}) is not None:
- 
 
 88 
                inStock = 0
- 
 
 89 
            else:
- 
 
 90 
                inStock = 1
83 
            return {'lowestSp':0,'inStock':0}
 91 
            return {'lowestSp':sellingPrice,'inStock':inStock,'buyBoxPrice':buyBoxPrice}
84 
 
 92 
 
85 
if __name__ == '__main__':
 93 
if __name__ == '__main__':
86 
    scraper = FlipkartProductPageScraper()
 94 
    scraper = FlipkartProductPageScraper()
87 
    print scraper.read('http://www.flipkart.com/spice-stellar-glide-mi-438/p/itme5kw5jx6jgskn?pid=MOBE2U6Y5WHBHMZZ')
 95 
    print scraper.read('http://www.flipkart.com/samsung-galaxy-star-advance/p/itmeyfc4vwzwhuva?pid=MOBEYFC44EUDZ9TX')