Subversion Repositories SmartDukaan

import urllib2

from BeautifulSoup import BeautifulSoup

import re

from sys import exit 

        print "Inside fetch details"

        flipkartSerialNumber = val.fkSerialNumber

        url = "http://www.flipkart.com/ps/%s"%(flipkartSerialNumber)

        #url = "http://www.flipkart.com/ps/MOBDTXVZXVY3GFG8"

        #scraper.read(url)

        #    scraper = FlipkartScraper.FlipkartScraper()

        #    vendorsData = scraper.read(url)

        request = urllib2.Request(url)

        request.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.218 Safari/535.1')

        opener = urllib2.build_opener()

        response_data = ""

        try:

            response_data = opener.open(request).read()

            print datetime.now()

            print "Fetched response from flipkart for %s" %(url)

        except urllib2.HTTPError as e:

            print 'ERROR: ', e

            print 'Retrying'

        response_data=response_data

        page=response_data.decode("utf-8")

        print datetime.now()

        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

        page = None

        response_data = None

        print datetime.now()

        print "Soup created from flipkart data for %s" %(url)

        info = []

        oddSeller = soup.findAll("div" , {"class" : "line seller-item odd "})

        for data in oddSeller:

            temp={}

            try:

                businessDays = data.find('span', attrs={'class' : re.compile('fk-deliverable.*')})

                shippingTime = businessDays.find('span', attrs={'class' : re.compile('fk-bold')}).string.replace('to','').replace('business days.','').strip().replace('  ','-')

                temp['shippingTime']=shippingTime

            except:

                pass

            price = data.find('span', attrs={'class' : re.compile('pxs-final-price.*')}).string.strip('Rs.').strip()

            temp['sellingPrice']=float(price)

            for sellerInfo in data.findAll("div",{"class":re.compile(".*seller-info*")}):

                sellerName = sellerInfo.find('a').string

                temp['sellerName'] = sellerName

            for metrics in data.find("div",{"class":"fk-text-right"}):

                try:

                    metric = metrics.findAll('input', {'type': 'submit'})

                except AttributeError:

                    continue

                try:

                    inputTags = metric[0]['data-lst-buytrend']

                except TypeError:

                    continue

                dataMetrics = metric[0]['data-listing-metrics']

                try:

                    buyTrend = inputTags[0:str(inputTags).index('NWSR')].replace('_','')

                except ValueError:

                    buyTrend = inputTags[0:str(inputTags).index('WSR')].replace('_','')

                temp['buyTrend']=buyTrend

                dataMetric = dataMetrics.split(';')

                sellerCode = dataMetric[0]

                temp['sellerCode']=sellerCode

                temp['sellingPriceMetric'] = float(dataMetric[1])

                if not temp.has_key('shippingTime'):

                    print "Populating shipping time from metrics"

                    temp['shippingTime'] = dataMetric[3]

                temp['sellerScore'] = int(dataMetric[4])

                info.append(temp)

        evenSeller = soup.findAll("div" , {"class" : "line seller-item even "})

        for data in evenSeller:

            temp={}

            price = data.find('span', attrs={'class' : re.compile('pxs-final-price.*')}).string.strip('Rs.')

            try:

                businessDays = data.find('span', attrs={'class' : re.compile('fk-deliverable.*')})

                shippingTime = businessDays.find('span', attrs={'class' : re.compile('fk-bold')}).string.replace('to','').replace('business days.','').strip().replace('  ','-')

                temp['shippingTime']=shippingTime

            except:

                pass

            temp['sellingPrice']=float(price)

            for sellerInfo in data.findAll("div",{"class":re.compile(".*seller-info*")}):

                sellerName = sellerInfo.find('a').string

                temp['sellerName'] = sellerName

            for metrics in data.find("div",{"class":"fk-text-right"}):

                try:

                    metric = metrics.findAll('input', {'type': 'submit'})

                except AttributeError:

                    continue

                try:

                    inputTags = metric[0]['data-lst-buytrend']

                except TypeError:

                    continue

                dataMetrics = metric[0]['data-listing-metrics']

                try:

                    buyTrend = inputTags[0:str(inputTags).index('NWSR')].replace('_','')

                except ValueError:

                    buyTrend = inputTags[0:str(inputTags).index('WSR')].replace('_','')

                temp['buyTrend']=buyTrend

                dataMetric = dataMetrics.split(';')

                temp['sellerCode'] = dataMetric[0] 

                temp['sellingPriceMetric'] = float(dataMetric[1])

                if not temp.has_key('shippingTime'):

                    print "Populating shipping time from metrics"

                    temp['shippingTime'] = dataMetric[3]

                temp['sellerScore'] = int(dataMetric[4])

                info.append(temp)

        vendorsData = info

        info = []

        print "Returning Json response from flipkart for %s" %(url)

        fin = datetime.now()

        print "Finish with data for serial Number %s %s" %(flipkartSerialNumber,str(fin))

        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))

        vendorsData[:]=[]

        rank ,ourSp, iterator, secondLowestSellerSp, prefSellerSp, lowestSellerSp, lowestSellerScore, prefSellerScore, secondLowestSellerScore, ourScore, shippingTimeLowerLimitLowestSeller,shippingTimeUpperLimitLowestSeller, \

        shippingTimeLowerLimitPrefSeller, shippingTimeUpperLimitPrefSeller, shippingTimeLowerLimitOur, shippingTimeUpperLimitOur, shippingTimeLowerLimitSecondLowestSeller, shippingTimeUpperLimitSecondLowestSeller, totalAvailableSeller= (0,)*19

        lowestSellerName, lowestSellerCode, secondLowestSellerName, secondLowestSellerCode, prefSellerName, prefSellerCode, lowestSellerBuyTrend, \

        ourBuyTrend, prefSellerBuyTrend, secondLowestSellerBuyTrend, ourCode = ('',)*11

        for data in sortedVendorsData:

            if iterator == 0:

                lowestSellerName = data['sellerName']

                lowestSellerScore = data['sellerScore']

                lowestSellerCode = data['sellerCode']

                lowestSellerSp = data['sellingPrice']

                lowestSellerBuyTrend = data['buyTrend']

                try:

                    shippingTimeLowerLimitLowestSeller, shippingTimeUpperLimitLowestSeller = data['shippingTime'].split('-')

                except ValueError:

                    shippingTimeLowerLimitLowestSeller = int(data['shippingTime'])

            if iterator ==1:

                secondLowestSellerName = data['sellerName']

                secondLowestSellerScore = data['sellerScore']

                secondLowestSellerCode = data['sellerCode']

                secondLowestSellerSp = data['sellingPrice']

                secondLowestSellerBuyTrend = data['buyTrend']

                try:

                    shippingTimeLowerLimitSecondLowestSeller, shippingTimeUpperLimitSecondLowestSeller = data['shippingTime'].split('-')

                except ValueError:

                    shippingTimeLowerLimitSecondLowestSeller = int(data['shippingTime'])

            if data['sellerName'] == 'Saholic':

                ourScore = data['sellerScore']

                ourCode = data['sellerCode']

                ourSp = data['sellingPrice']

                ourBuyTrend = data['buyTrend']

                try:

                    shippingTimeLowerLimitOur, shippingTimeUpperLimitOur = data['shippingTime'].split('-')

                except ValueError:

                    shippingTimeLowerLimitOur = int(data['shippingTime'])

                rank = iterator + 1

            if data['buyTrend'] in ('PrefCheap','PrefNCheap',''):

                prefSellerName = data['sellerName']

                prefSellerScore = data['sellerScore']

                prefSellerCode = data['sellerCode']

                prefSellerSp = data['sellingPrice']

                prefSellerBuyTrend = data['buyTrend']

                try:

                    shippingTimeLowerLimitPrefSeller, shippingTimeUpperLimitPrefSeller = data['shippingTime'].split('-')

                except ValueError:

                    shippingTimeLowerLimitPrefSeller = int(data['shippingTime'])

            iterator+=1

        flipkartDetails = __FlipkartDetails(rank ,ourSp , secondLowestSellerSp, prefSellerSp, lowestSellerSp, lowestSellerScore, prefSellerScore, secondLowestSellerScore, ourScore, int(shippingTimeLowerLimitLowestSeller),int(shippingTimeUpperLimitLowestSeller), \

        ourBuyTrend, prefSellerBuyTrend, secondLowestSellerBuyTrend, ourCode)

        if flipkartDetails.ourBuyTrend == 'PrefCheap'and flipkartDetails.rank!=1 and flipkartDetails.ourSp==flipkartDetails.secondLowestSellerSp:

            print "Under PrefCheap category.Switching data for ",flipkartSerialNumber

            flipkartDetails.lowestSellerSp, flipkartDetails.secondLowestSellerSp = flipkartDetails.secondLowestSellerSp,flipkartDetails.lowestSellerSp

            flipkartDetails.lowestSellerScore, flipkartDetails.secondLowestSellerScore = flipkartDetails.secondLowestSellerScore,flipkartDetails.lowestSellerScore

            flipkartDetails.shippingTimeLowerLimitLowestSeller, flipkartDetails.shippingTimeLowerLimitSecondLowestSeller = flipkartDetails.shippingTimeLowerLimitSecondLowestSeller,flipkartDetails.shippingTimeLowerLimitLowestSeller

            flipkartDetails.shippingTimeUpperLimitLowestSeller, flipkartDetails.shippingTimeUpperLimitSecondLowestSeller = flipkartDetails.shippingTimeUpperLimitSecondLowestSeller,flipkartDetails.shippingTimeUpperLimitLowestSeller

            flipkartDetails.lowestSellerName, flipkartDetails.secondLowestSellerName = flipkartDetails.secondLowestSellerName,flipkartDetails.lowestSellerName

            flipkartDetails.lowestSellerCode, flipkartDetails.secondLowestSellerCode = flipkartDetails.secondLowestSellerCode,flipkartDetails.lowestSellerCode

            flipkartDetails.lowestSellerBuyTrend, flipkartDetails.secondLowestSellerBuyTrend = flipkartDetails.secondLowestSellerBuyTrend,flipkartDetails.lowestSellerBuyTrend

            flipkartDetails.rank=1

        if flipkartDetails.ourBuyTrend == 'NPrefCheap'and flipkartDetails.rank!=1 and flipkartDetails.ourSp==flipkartDetails.lowestSellerSp:

            print "Under NPrefCheap category.Switching data for ",flipkartSerialNumber

            flipkartDetails.lowestSellerSp = flipkartDetails.ourSp

            flipkartDetails.lowestSellerScore = flipkartDetails.ourScore

            flipkartDetails.shippingTimeLowerLimitLowestSeller = flipkartDetails.shippingTimeLowerLimitOur

            flipkartDetails.shippingTimeUpperLimitLowestSeller = flipkartDetails.shippingTimeUpperLimitOur

            flipkartDetails.lowestSellerName = 'Saholic'

            flipkartDetails.lowestSellerCode = flipkartDetails.ourCode

            flipkartDetails.lowestSellerBuyTrend = flipkartDetails.ourBuyTrend

            #flipkartDetails = fetchDetails(val.fkSerialNumber)