Subversion Repositories SmartDukaan

import pymongo
from dtr.utils.utils import to_java_date
from datetime import datetime, timedelta
from operator import itemgetter
from dtr.utils import FlipkartScraper,NewFlipkartScraper
from multiprocessing import Pool as ThreadPool
from multiprocessing import cpu_count
import optparse
from dtr.storage.MemCache import MemCache

con = None

parser = optparse.OptionParser()
parser.add_option("-m", "--m", dest="mongoHost",
                      default="localhost",
                      type="string", help="The HOST where the mongo server is running",
                      metavar="mongo_host")

(options, args) = parser.parse_args()

mc = MemCache(options.mongoHost)

def get_mongo_connection(host=options.mongoHost, port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def populate():
    toScrapMap = {}
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
    for bestSeller in bestSellers: 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
        for data in amazonBestSellers:
            if not toScrapMap.has_key(data['_id']):
                toScrapMap[data['_id']] = data
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':2,'showDeal':1,'totalPoints':{'$gt':0}}))
    for deal in dealFlagged:
        if not toScrapMap.has_key(deal['_id']):
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
            toScrapMap[deal['_id']] = data[0]
    pool = ThreadPool(cpu_count() *2)
    pool.map(scrapeFlipkart,toScrapMap.values())
    pool.close()
    pool.join()
    print "joining threads at %s"%(str(datetime.now()))

def scrapeFlipkart(data):
    if data['source_id']!=2:
        return
    retryCount = 0
    print str(data['identifier'])
    if data['identifier'] is None or len(data['identifier'].strip())==0:
        print "returning in valid identifier"
        return
    
    try:
        if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
            print "sku id is already updated",data['_id'] 
            return
    except:
        pass
    
    
    lowestSp = 0
    inStock = 0
    scraperFk = FlipkartScraper.FlipkartScraper()
    scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
    try:
        if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
            result = scraperProductPage.read(data['marketPlaceUrl'])
            if result.get('lowestSp')!=0:
                lowestSp = result.get('lowestSp')
                inStock = result.get('inStock')
    except:
        print "Unable to scrape product page ",data['identifier']
    
    
    if lowestSp == 0:
        url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
        while(retryCount < 3):
            try:
                vendorsData = scraperFk.read(url)
                fetched = True
                break
            except Exception as e:
                print "***Retry count ",retryCount 
                retryCount+=1
                if retryCount == 3:
                    fetched = False
                print e
        if not fetched:
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
            return
        
        sortedVendorsData = []
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
        print "data",sortedVendorsData
        lowestSp, iterator = (0,)*2
        for vData in sortedVendorsData:
            if iterator == 0:
                lowestSp = vData['sellingPrice']
            break
        if lowestSp > 0:
            inStock = 1
    print lowestSp
    print inStock
    if lowestSp > 0:
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
    else:
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
        
    try:
        recomputeDeal(data['skuBundleId'])
    except:
        print "Unable to compute deal for ",data['skuBundleId']

def populateNegativeDeals():
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
    mc.set("negative_deals", negativeDeals, 600)  

def recomputeDeal(skuBundleId):
    """Lets recompute deal for this bundle"""
    print "Recomputing for bundleId",skuBundleId
    
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
    bestPrice = float("inf")
    bestOne = None
    bestSellerPoints = 0
    toUpdate = []
    for similarItem in similarItems:
        if mc.get("negative_deals") is None:
            populateNegativeDeals()
        if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
            continue
        if similarItem['available_price'] < bestPrice:
            bestOne = similarItem
            bestPrice = similarItem['available_price']
            bestSellerPoints = similarItem['bestSellerPoints']
        elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
            bestOne = similarItem
            bestPrice = similarItem['available_price']
            bestSellerPoints = similarItem['bestSellerPoints']
        else:
            pass
    if bestOne is not None:
        for similarItem in similarItems:
            toUpdate.append(similarItem['_id'])
        toUpdate.remove(bestOne['_id'])
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
    if len(toUpdate) > 0:
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)

def main():
    populate()
            
if __name__=='__main__':
    main()