Subversion Repositories SmartDukaan

import pymongo

from dtr.utils.utils import to_java_date

from datetime import datetime

from operator import itemgetter

from dtr.utils import FlipkartScraper

con = None

now = datetime.now()

print to_java_date(now)

scraperFk = FlipkartScraper.FlipkartScraper()

def get_mongo_connection(host='localhost', port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def scrapeFlipkart():

    flipkartBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':2}))

    for data in flipkartBestSellers:

        inStock = 0

        retryCount = 0

        print str(data['identifier'])

        if data['identifier'] is None or len(data['identifier'].strip())==0:

            print "continue"

            continue

        url = "http://www.flipkart.com/ps/%s"%(data['identifier'])

        while(retryCount < 3):

            try:

                vendorsData = scraperFk.read(url)

                fetched = True

            except Exception as e:

                print "***Retry count ",retryCount 

                retryCount+=1

                if retryCount == 3:

                    fetched = False

                print e

        if not fetched:

            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']

            continue

        sortedVendorsData = []

        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))

        print "data",sortedVendorsData

        lowestSp, iterator = (0,)*2

        for vData in sortedVendorsData:

            if iterator == 0:

                lowestSp = vData['sellingPrice']

            break

        if lowestSp > 0:

            inStock = 1

        print lowestSp

        print inStock

        if lowestSp > 0:

            get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

        else:

            get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

        print "+++++++++++++++++++++++++++"

def main():

    scrapeFlipkart()

if __name__=='__main__':

    main()