Subversion Repositories SmartDukaan

import urllib2

import simplejson as json

import pymongo

from dtr.utils.utils import to_java_date

from datetime import datetime, timedelta

from operator import itemgetter

from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper

from dtr.utils import AmazonDealScraper

from dtr.utils import FlipkartScraper,NewFlipkartScraper

from dtr.storage.MemCache import MemCache

mc = MemCache("127.0.0.1")

con = None

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4}

headers = { 

           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'

        }

def get_mongo_connection(host='localhost', port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def returnLatestPrice(data, source_id):

    now = datetime.now()

    if source_id == 1:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            try:

                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=0)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

            except:

                pass

            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data['identifier'])

            lowestPrice = 0.0

            print data['dealFlag']

            print data['dealType']

            try:

                if data['dealFlag'] ==1 and data['dealType'] ==1:

                    print "Inside deal"

                    deal_url = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())

                    print deal_url

                    dealScraperAmazon = AmazonDealScraper.AmazonScraper()

                    lowestPrice = dealScraperAmazon.read(deal_url)

                    print lowestPrice

                    if lowestPrice == 0:

                        raise

                else:    

                    scraperAmazon = AmazonScraper()

                    lowestPrice = scraperAmazon.read(url)

            except Exception as e:

                print e

                scraperAmazon = AmazonScraper()

                lowestPrice = scraperAmazon.read(url)

            print "LowestPrice ",lowestPrice

            inStock = 0

            if lowestPrice > 0:

                inStock = 1

            if lowestPrice > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType']}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType']}}, multi=True)

            try:    

                recomputeDeal(data['skuBundleId'])

            except:

                print "Unable to compute deal for ",data['skuBundleId']

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

        except:

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

    elif source_id ==4:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            try:

                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id'] 

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

            except:

                pass

            url = "http://109.74.200.220:8080/mobileapi/dtr-pricing?id=%s"%(data['identifier'])

            lowestPrice = 0.0

            instock = 0

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            json_input = response.read()

            response.close()

            priceInfo = json.loads(json_input)

            lowestPrice = priceInfo['response']['sellingPrice']

            if lowestPrice > 0:

                instock = 1

            if instock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':instock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':instock}}, multi=True)

            else:

                lowestPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock}}, multi=True)

            try:    

                recomputeDeal(data['skuBundleId'])

            except:

                print "Unable to compute deal for ",data['skuBundleId']

            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

        except:

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

    elif source_id ==3:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            try:

                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

            except Exception as e:

                print "Exception snapdeal"

                print e

                pass

            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(data['identifier'])

            req = urllib2.Request(url,headers=headers)

            response = urllib2.urlopen(req)

            json_input = response.read()

            response.close()

            vendorInfo = json.loads(json_input)

            lowestOfferPrice = 0

            inStock = 0

            for vendor in vendorInfo:

                lowestOfferPrice = float(vendor['sellingPrice'])

                stock = vendor['buyableInventory']

                if stock > 0 and lowestOfferPrice > 0:

                    inStock = 1

                    break

            print lowestOfferPrice

            print inStock

            print "*************"

            if inStock  == 1:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock}}, multi=True)

            else:

                lowestOfferPrice = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)

            try:    

                recomputeDeal(data['skuBundleId'])

            except:

                print "Unable to compute deal for ",data['skuBundleId']

            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

        except:

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

    elif source_id == 2:

        try:

            if data['identifier'] is None or len(data['identifier'].strip())==0:

                return {}

            try:

                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):

                    print "sku id is already updated",data['_id']

                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']} 

            except:

                pass

            lowestSp = 0

            inStock = 0

            scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()

            try:

                if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):

                    result = scraperProductPage.read(data['marketPlaceUrl'])

                    print result

                    if result.get('lowestSp')!=0:

                        lowestSp = result.get('lowestSp')

                        inStock = result.get('inStock')

            except:

                print "Unable to scrape product page ",data['identifier']

            if lowestSp ==0:

                url = "http://www.flipkart.com/ps/%s"%(data['identifier'])

                scraperFk = FlipkartScraper.FlipkartScraper()

                vendorsData = scraperFk.read(url)

                sortedVendorsData = []

                sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))

                print "data",sortedVendorsData

                for vData in sortedVendorsData:

                    lowestSp = vData['sellingPrice']

                    break

                if lowestSp > 0:

                    inStock = 1

            print lowestSp

            print inStock

            if lowestSp > 0:

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)

            else:

                lowestSp = data['available_price']

                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)

                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)

            try:    

                recomputeDeal(data['skuBundleId'])

            except:

                print "Unable to compute deal for ",data['skuBundleId']

            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

        except:

            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}

    else:

        return {}

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)

def recomputeDeal(skuBundleId):

    """Lets recompute deal for this bundle"""

    print "Recomputing for bundleId",skuBundleId

    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))

    bestPrice = float("inf")

    bestOne = None

    bestSellerPoints = 0

    toUpdate = []

    for similarItem in similarItems:

        if mc.get("negativeDeals") is None:

            populateNegativeDeals()

        if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negativeDeals"):

            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})

            continue

        if similarItem['available_price'] < bestPrice:

            bestOne = similarItem

            bestPrice = similarItem['available_price']

            bestSellerPoints = similarItem['bestSellerPoints']

        elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:

            bestOne = similarItem

            bestPrice = similarItem['available_price']

            bestSellerPoints = similarItem['bestSellerPoints']

        else:

            pass

    if bestOne is not None:

        for similarItem in similarItems:

            toUpdate.append(similarItem['_id'])

        toUpdate.remove(bestOne['_id'])

        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})

    if len(toUpdate) > 0:

        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)

def getLatestPrice(skuBundleId, source_id):

    temp = []

    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))

    for item in itemIds:

        item['dealFlag'] = 0

        item['dealType'] = 0

        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))

        if len(manualDeals) > 0:

            item['dealFlag'] = 1

            item['dealType'] =manualDeals[0]['dealType'] 

        temp.append(returnLatestPrice(item, source_id))

    return temp

def getLatestPriceById(id):

    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))

    item[0]['dealFlag'] = 0

    item[0]['dealType'] = 0

    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item[0]['source_id'], 'sku':item[0]['_id']}))

    if len(manualDeals) > 0:

        item[0]['dealFlag'] = 1

        item[0]['dealType'] =manualDeals[0]['dealType'] 

    return returnLatestPrice(item[0], item[0]['source_id'])

def main():

    print getLatestPriceById(22746)

if __name__=='__main__':

    main()

"""21.06$"""