Subversion Repositories SmartDukaan

import urllib2

import simplejson as json

import pymongo

from dtr.utils.utils import to_java_date

from datetime import datetime, timedelta

from operator import itemgetter

from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper

from dtr.utils import AmazonDealScraper

from dtr.utils import FlipkartScraper,NewFlipkartScraper

from dtr.storage.MemCache import MemCache

from functools import partial

import threading

from dtr.utils.utils import getCashBack

mc = MemCache("127.0.0.1")

con = None

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4}

headers = { 

           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'

        }

def get_mongo_connection(host='localhost', port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def returnLatestPrice(data, source_id):

    now = datetime.now()

    if source_id == 1:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            if data['dealFlag'] ==1 and data['dealType'] ==1:

                data['marketPlaceUrl'] = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())

            try:

                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

            except:

                pass

            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data['identifier'])

            lowestPrice = 0.0

            try:

                if data['dealFlag'] ==1 and data['dealType'] ==1:

                    print "Inside deal"

                    deal_url = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())

                    print deal_url

                    dealScraperAmazon = AmazonDealScraper.AmazonScraper()

                    lowestPrice = dealScraperAmazon.read(deal_url)

                    print lowestPrice

                    if lowestPrice == 0:

                        raise

                else:    

                    scraperAmazon = AmazonScraper()

                    lowestPrice = scraperAmazon.read(url)

            except Exception as e:

                print e

                scraperAmazon = AmazonScraper()

                lowestPrice = scraperAmazon.read(url)

            print "LowestPrice ",lowestPrice

            inStock = 0

            if lowestPrice > 0:

                inStock = 1

            if lowestPrice > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType']}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType']}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

        except:

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

    elif source_id ==4:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            try:

                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

            except:

                pass

            url = "http://109.74.200.220:8080/mobileapi/dtr-pricing?id=%s"%(data['identifier'])

            lowestPrice = 0.0

            instock = 0

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            json_input = response.read()

            response.close()

            priceInfo = json.loads(json_input)

            lowestPrice = priceInfo['response']['sellingPrice']

            if lowestPrice > 0:

                instock = 1

            if instock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':instock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':instock}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

        except:

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

    elif source_id ==3:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            try:

                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

            except Exception as e:

                print "Exception snapdeal"

                print e

                pass

            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(data['identifier'])

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            json_input = response.read()

            response.close()

            vendorInfo = json.loads(json_input)

            lowestOfferPrice = 0

            inStock = 0

            for vendor in vendorInfo:

                lowestOfferPrice = float(vendor['sellingPrice'])

                stock = vendor['buyableInventory']

                if stock > 0 and lowestOfferPrice > 0:

                    inStock = 1

                    break

            print lowestOfferPrice

            print inStock

            print "*************"

            if inStock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock}}, multi=True)

            else:

                lowestOfferPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

        except:

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

    elif source_id == 2:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            try:

                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']} 

            except:

                pass

            lowestSp = 0

            inStock = 0

            scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()

            try:

                if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):

                    result = scraperProductPage.read(data['marketPlaceUrl'])

                    print result

                    if result.get('lowestSp')!=0:

                        lowestSp = result.get('lowestSp')

                        inStock = result.get('inStock')

            except:

                print "Unable to scrape product page ",data['identifier']

            if lowestSp ==0:

                url = "http://www.flipkart.com/ps/%s"%(data['identifier'])

                scraperFk = FlipkartScraper.FlipkartScraper()

                vendorsData = scraperFk.read(url)

                sortedVendorsData = []

                sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))

                print "data",sortedVendorsData

                for vData in sortedVendorsData:

                    lowestSp = vData['sellingPrice']

                    break

                if lowestSp > 0:

                    inStock = 1

            print lowestSp

            print inStock

            if lowestSp > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)

            else:

                lowestSp = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)

            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

        except:

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

    else:

        return {}

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)

def recomputeDeal(skuBundleId):

    """Lets recompute deal for this bundle"""

    print "Recomputing for bundleId",skuBundleId

    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))

    bestPrice = float("inf")

    bestOne = None

    bestSellerPoints = 0

    toUpdate = []

    for similarItem in similarItems:

        if mc.get("negative_deals") is None:

            populateNegativeDeals()

        if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):

            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})

            continue

        if similarItem['available_price'] < bestPrice:

            bestOne = similarItem

            bestPrice = similarItem['available_price']

            bestSellerPoints = similarItem['bestSellerPoints']

        elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:

            bestOne = similarItem

            bestPrice = similarItem['available_price']

            bestSellerPoints = similarItem['bestSellerPoints']

        else:

            pass

    if bestOne is not None:

        for similarItem in similarItems:

            toUpdate.append(similarItem['_id'])

        toUpdate.remove(bestOne['_id'])

        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})

    if len(toUpdate) > 0:

        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)

    print "Done with recomputing"

def getLatestPrice(skuBundleId, source_id):

    temp = []

    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))

    for item in itemIds:

        item['dealFlag'] = 0

        item['dealType'] = 0

        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))

        if len(manualDeals) > 0:

            item['dealFlag'] = 1

            item['dealType'] =manualDeals[0]['dealType']

        info = returnLatestPrice(item, source_id)

        try:

            cashBack = getCashBack(item['_id'], item['source_id'], item['category_id'], mc, 'localhost')

            if not cashBack or cashBack.get('cash_back_status')!=1:

                pass

            else:

                if cashBack['cash_back_type'] in (1,2):

                    info['cash_back_type'] = cashBack['cash_back_type']

                    info['cash_back'] = float(cashBack['cash_back'])

                else:

                    raise

        except Exception as cashBackEx:

            print cashBackEx

            print "Error calculating cashback."

            info['cash_back_type'] = 0

            info['cash_back'] = 0

        temp.append(info)

    try:

        thread = threading.Thread(target=recomputeDeal, args = (skuBundleId,))

        thread.daemon = True

        thread.start()    

    except:

        print "Unable to compute deal for ",skuBundleId

    return temp

def getLatestPriceById(id):

    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))

    item[0]['dealFlag'] = 0

    item[0]['dealType'] = 0

    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item[0]['source_id'], 'sku':item[0]['_id']}))

    if len(manualDeals) > 0:

        item[0]['dealFlag'] = 1

        item[0]['dealType'] =manualDeals[0]['dealType'] 

    info = returnLatestPrice(item[0], item[0]['source_id'])

    try:

        cashBack = getCashBack(item['_id'], item['source_id'], item['category_id'], mc, 'localhost')

        if not cashBack or cashBack.get('cash_back_status')!=1:

            pass

        else:

            if cashBack['cash_back_type'] in (1,2):

                info['cash_back_type'] = cashBack['cash_back_type']

                info['cash_back'] = float(cashBack['cash_back'])

            else:

                raise

    except Exception as cashBackEx:

        print cashBackEx

        print "Error calculating cashback."

        info['cash_back_type'] = 0

        info['cash_back'] = 0

    try:

        thread = threading.Thread(target=recomputeDeal, args = (item[0]['skuBundleId'],))

        thread.daemon = True

        thread.start()    

    except:

        print "Unable to compute deal for ",item[0]['skuBundleId']

    return info

def main():

    print "retuned %s"%(str(getLatestPrice(17, 2)))

if __name__=='__main__':

    main()