Subversion Repositories SmartDukaan

import urllib2
import simplejson as json
import pymongo
from dtr.utils.utils import to_java_date, getNlcPoints 
from datetime import datetime, timedelta
import time
from multiprocessing import Pool as ThreadPool
from multiprocessing import cpu_count
import optparse
from dtr.storage.MemCache import MemCache
from dtr.utils.utils import getCashBack, get_mongo_connection, SOURCE_MAP
import traceback
from operator import itemgetter
import chardet
from dtr.utils import HomeShop18Scraper


SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5,'PAYTM.COM':6, 'HOMESHOP18.COM':7}

con = None

parser = optparse.OptionParser()
parser.add_option("-m", "--m", dest="mongoHost",
                      default="localhost",
                      type="string", help="The HOST where the mongo server is running",
                      metavar="mongo_host")

(options, args) = parser.parse_args()

mc = MemCache(options.mongoHost)

def getNetPriceForItem(itemId, source_id, category_id ,price):
    cash_back_type = 0
    cash_back = 0
    try:
        cashBack = getCashBack(itemId, source_id, category_id, mc, options.mongoHost)
        if not cashBack or cashBack.get('cash_back_status')!=1:
            cash_back_type = 0
            cash_back = 0 
            
        else:
            if cashBack['cash_back_type'] in (1,2):
                
                if cashBack.get('maxCashBack') is not None:
                    
                    if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*price)/100 > cashBack.get('maxCashBack'):
                        cashBack['cash_back_type'] = 2
                        cashBack['cash_back'] = cashBack['maxCashBack']
                    elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):
                        cashBack['cash_back'] = cashBack['maxCashBack']
                    else:
                        pass
                
                
                
                cash_back_type = cashBack['cash_back_type']
                cash_back = float(cashBack['cash_back'])
    except Exception as cashBackEx:
        pass
    
    if cash_back_type ==1:
        return (price - float(cash_back)*price/100)
    elif cash_back_type ==2:
        return (price - cash_back)
    else:
        return price


def populate():
    toScrapMap = {}
    bestSellers = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'rank':{'$gt':0}}))
    for bestSeller in bestSellers: 
        snapdealBestSellers = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':7}))
        for data in snapdealBestSellers:
            if not toScrapMap.has_key(data['_id']):
                data['dealFlag'] = 0
                data['dealType'] = 0
                toScrapMap[data['_id']] = data
    dealFlagged = list(get_mongo_connection(host=options.mongoHost).Catalog.Deals.find({'source_id':7,'showDeal':1,'totalPoints':{'$gt':-100}}))
    for deal in dealFlagged:
        if not toScrapMap.has_key(deal['_id']):
            data = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'_id':deal['_id']}))
            data[0]['dealFlag'] = 0
            data[0]['dealType'] = 0
            toScrapMap[deal['_id']] = data[0]
    manualDeals = list(get_mongo_connection(host=options.mongoHost).Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':7}))
    for manualDeal in manualDeals:
        if not toScrapMap.has_key(manualDeal['sku']):
            data = list(get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'_id':manualDeal['sku']}))
            if len(data) > 0:
                data[0]['dealFlag'] = 1
                data[0]['dealType'] = manualDeal['dealType']
                toScrapMap[manualDeal['sku']] = data[0]
        else:
            data = toScrapMap.get(manualDeal['sku'])
            data['dealFlag'] = 1
            data['dealType'] = manualDeal['dealType']
    
    for val in toScrapMap.values():
        updatePrices(val)
        
def updatePrices(data):
    if data.get('ignorePricing') ==1:
        print "Ignored items returning for %d"%(data['_id'])
        return
    if data['source_id']!=7:
        return
    print data['identifier']
    if data['identifier'] is None or len(data['identifier'].strip())==0:
        print "returning"
        return
    
    try:
        if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
            print "sku id is already updated",data['_id'] 
            return
    except:
        pass
    
    result = None
    try:
        url = 'http://m.homeshop18.com/product.mobi?productId='+str(data['identifier'])
        scraper = HomeShop18Scraper.HomeShop18Scraper()
        result = scraper.read(url)
    except:
        print "Unable to scrape %d"%(data['_id'])
        get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now())}})
        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})
        return
    
    inStock = 0
    
    lowestOfferPrice = 0
    if result is not None:
        lowestOfferPrice = float(result['price']+result['shippingCharge'])
        inStock = result['inStock']
        
    print lowestOfferPrice
    print inStock
    print "*************"
    
    if lowestOfferPrice ==0:
        inStock = 0
        
    if inStock  == 1:
        netPriceAfterCashBack = getNetPriceForItem(data['_id'], SOURCE_MAP.get('HOMESHOP18.COM'), data['category_id'], lowestOfferPrice)
        get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock,'codAvailable':data['codAvailable'],'netPriceAfterCashBack':netPriceAfterCashBack}}, multi=True)
    else:
        lowestOfferPrice = data['available_price']
        netPriceAfterCashBack = getNetPriceForItem(data['_id'], SOURCE_MAP.get('HOMESHOP18.COM'), data['category_id'], lowestOfferPrice)
        get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
        get_mongo_connection(host=options.mongoHost).Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock,'codAvailable':data['codAvailable'],'netPriceAfterCashBack':netPriceAfterCashBack}}, multi=True)
        
    try:
        recomputeDeal(data)
    except:
        print "Unable to compute deal for ",data['skuBundleId']
        
def populateNegativeDeals():
    negativeDeals = get_mongo_connection(host=options.mongoHost).Catalog.NegativeDeals.find().distinct('sku')
    mc.set("negative_deals", negativeDeals, 600)
    
def recomputeDeal(item):
    """Lets recompute deal for this bundle"""
    print "Recomputing for bundleId %d" %(item.get('skuBundleId'))
    skuBundleId = item['skuBundleId']
    
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('netPriceAfterCashBack',pymongo.ASCENDING)]))
    bestPrice = float("inf")
    bestOne = None
    bestSellerPoints = 0
    toUpdate = []
    prepaidBestPrice = float("inf")
    prepaidBestOne = None
    prepaidBestSellerPoints = 0
    for similarItem in similarItems:
        if similarItem['codAvailable'] ==1:
            if mc.get("negative_deals") is None:
                populateNegativeDeals()
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                continue
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                continue
            if similarItem.get('netPriceAfterCashBack') < bestPrice:
                bestOne = similarItem
                bestPrice = similarItem.get('netPriceAfterCashBack')
                bestSellerPoints = similarItem['bestSellerPoints']
            elif similarItem.get('netPriceAfterCashBack') == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
                bestOne = similarItem
                bestPrice = similarItem.get('netPriceAfterCashBack')
                bestSellerPoints = similarItem['bestSellerPoints']
            else:
                pass
        else:
            if mc.get("negative_deals") is None:
                populateNegativeDeals()
            if similarItem['in_stock'] == 0  or similarItem['_id'] in mc.get("negative_deals"):
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
                continue
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
                continue
            if similarItem.get('netPriceAfterCashBack') < prepaidBestPrice:
                prepaidBestOne = similarItem
                prepaidBestPrice = similarItem.get('netPriceAfterCashBack')
                prepaidBestSellerPoints = similarItem['bestSellerPoints']
            elif similarItem.get('netPriceAfterCashBack') == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
                prepaidBestOne = similarItem
                prepaidBestPrice = similarItem.get('netPriceAfterCashBack')
                prepaidBestSellerPoints = similarItem['bestSellerPoints']
            else:
                pass
    if bestOne is not None or prepaidBestOne is not None:
        for similarItem in similarItems:
            toUpdate.append(similarItem['_id'])
        if bestOne is not None:
            toUpdate.remove(bestOne['_id'])
            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
        if prepaidBestOne is not None:
            if bestOne is not None:
                if prepaidBestOne.get('netPriceAfterCashBack') < bestOne.get('netPriceAfterCashBack'): 
                    toUpdate.remove(prepaidBestOne['_id'])
                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
            else:
                toUpdate.remove(prepaidBestOne['_id'])
                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
    if len(toUpdate) > 0:
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)


def main():
    populate()

if __name__=='__main__':
    main()