Subversion Repositories SmartDukaan

import pymongo
from dtr.utils.utils import to_java_date
from datetime import datetime
from operator import itemgetter
from dtr.utils import FlipkartScraper

con = None
now = datetime.now()
print to_java_date(now)
scraperFk = FlipkartScraper.FlipkartScraper()

def get_mongo_connection(host='localhost', port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def scrapeFlipkart():
    flipkartBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':2}))
    for data in flipkartBestSellers:
        inStock = 0
        retryCount = 0
        print str(data['identifier'])
        if data['identifier'] is None or len(data['identifier'].strip())==0:
            print "continue"
            continue
        
        url = "http://www.flipkart.com/ps/%s"%(data['identifier'])
        while(retryCount < 3):
            try:
                vendorsData = scraperFk.read(url)
                fetched = True
                break
            except Exception as e:
                print "***Retry count ",retryCount 
                retryCount+=1
                if retryCount == 3:
                    fetched = False
                print e
        if not fetched:
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
            continue
        sortedVendorsData = []
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
        print "data",sortedVendorsData
        lowestSp, iterator = (0,)*2
        for vData in sortedVendorsData:
            if iterator == 0:
                lowestSp = vData['sellingPrice']
            break
        if lowestSp > 0:
            inStock = 1
        print lowestSp
        print inStock
        if lowestSp > 0:
            get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
        else:
            get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
        print "+++++++++++++++++++++++++++"
       
            
        

def main():
    scrapeFlipkart()
            
if __name__=='__main__':
    main()