Subversion Repositories SmartDukaan

Rev

Rev 15265 | Rev 17103 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
15950 kshitij.so 1 
from dtr.utils.utils import fetchResponseUsingProxy, transformUrl
14122 kshitij.so 2 
from sys import exit
3 
import json
14746 kshitij.so 4 
import re
15265 kshitij.so 5 
import traceback
15950 kshitij.so 6 
import datetime
7 
from pyquery import PyQuery
14122 kshitij.so 8 
 
14746 kshitij.so 9 
 
15950 kshitij.so 10 
 
14122 kshitij.so 11 
headers = {
12 
            'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',
13 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
14 
            'Accept-Language' : 'en-US,en;q=0.8',
15 
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
14746 kshitij.so 16 
            'Connection':'keep-alive',
17 
            'Accept-Encoding' : 'gzip,deflate,sdch'
14122 kshitij.so 18 
        }
19 
 
15950 kshitij.so 20 
 
14122 kshitij.so 21 
class FlipkartProductPageScraper:
22 
    def __init__(self):
23 
        self.count_trials = 0
24 
        self.redirectCount = 0
25 
 
26 
    def read(self, url):
27 
        response_data = ""
28 
        redirect_url = ""
15950 kshitij.so 29 
        url = transformUrl(url,2)
14122 kshitij.so 30 
        try:
14538 kshitij.so 31 
 
32 
            """quick fix,need to add it conf"""
33 
 
15950 kshitij.so 34 
            response_data = fetchResponseUsingProxy(url, headers, proxy=True)
14122 kshitij.so 35 
            print "Fetched response from flipkart for %s" %(url)
14538 kshitij.so 36 
            #redirect_url = response.url
14122 kshitij.so 37 
 
38 
        except Exception as e:
14746 kshitij.so 39 
            traceback.print_exc()
14122 kshitij.so 40 
            print 'ERROR: ', e
41 
            print 'Retrying'
42 
            self.count_trials += 1
43 
 
44 
            if self.count_trials < 3:
45 
                return self.read(url)
46 
 
47 
        self.response_data=response_data
15950 kshitij.so 48 
        print datetime.datetime.now()
14122 kshitij.so 49 
        return self.createData(url,redirect_url)
50 
 
51 
    def createData(self,url, redirect_url):
15950 kshitij.so 52 
        pq = PyQuery(self.response_data)
53 
        buyBoxPrice = float(pq('span.selling-price.omniture-field').attr['data-evar48'])
54 
        inStock = 1
14122 kshitij.so 55 
        try:
15950 kshitij.so 56 
            sellerJson = pq('div.seller-table-wrap').attr['data-config']
57 
            x = json.loads(sellerJson)
58 
            lines = sorted(x['dataModel'], key=lambda k: k['priceInfo'].get('sellingPrice', 0), reverse=False)
14122 kshitij.so 59 
            sellingPrice =  float(lines[0]['priceInfo']['sellingPrice'])
60 
            try:
61 
                offerText = lines[0]['offerInfo']['listingOffers'][0]['description']
62 
            except:
63 
                offerText = ""
15950 kshitij.so 64 
            return {'lowestSp':sellingPrice,'inStock':inStock,'buyBoxPrice':buyBoxPrice}
14122 kshitij.so 65 
        except:
15950 kshitij.so 66 
            """Not able to parse seller wrap section, probably due to only single seller option"""
15265 kshitij.so 67 
            sellingPrice = buyBoxPrice
15950 kshitij.so 68 
            stockDiv = pq('div.out-of-stock')
69 
            if len(stockDiv) > 0:
15265 kshitij.so 70 
                inStock = 0
71 
            return {'lowestSp':sellingPrice,'inStock':inStock,'buyBoxPrice':buyBoxPrice}
15950 kshitij.so 72 
 
14122 kshitij.so 73 
if __name__ == '__main__':
15950 kshitij.so 74 
    print datetime.datetime.now()
14122 kshitij.so 75 
    scraper = FlipkartProductPageScraper()
15950 kshitij.so 76 
 
77 
    print scraper.read('http://www.flipkart.com/redmi-2/p/itme8ygtcfax6w39')
78 
    print datetime.datetime.now()