Subversion Repositories SmartDukaan

Rev

Rev 14705 | Rev 14743 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed 

import urllib2
import simplejson as json
import pymongo
from dtr.utils.utils import to_java_date
from datetime import datetime, timedelta
from operator import itemgetter
from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper
from dtr.utils import AmazonDealScraper
from dtr.utils import FlipkartScraper,NewFlipkartScraper
from dtr.storage.MemCache import MemCache
from functools import partial
import threading

mc = MemCache("127.0.0.1")

con = None
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4}

headers = { 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      
            'Accept-Language' : 'en-US,en;q=0.8',                     
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
        }

def get_mongo_connection(host='localhost', port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def returnLatestPrice(data, source_id):
    now = datetime.now()
    if source_id == 1:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            if data['dealFlag'] ==1 and data['dealType'] ==1:
                data['marketPlaceUrl'] = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())
            
            try:
                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
            except:
                pass
            
            
            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data['identifier'])
            lowestPrice = 0.0
            print data['dealFlag']
            print data['dealType']
            try:
                if data['dealFlag'] ==1 and data['dealType'] ==1:
                    print "Inside deal"
                    deal_url = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())
                    print deal_url
                    dealScraperAmazon = AmazonDealScraper.AmazonScraper()
                    lowestPrice = dealScraperAmazon.read(deal_url)
                    print lowestPrice
                    if lowestPrice == 0:
                        raise
                else:    
                    scraperAmazon = AmazonScraper()
                    lowestPrice = scraperAmazon.read(url)
            except Exception as e:
                print e
                scraperAmazon = AmazonScraper()
                lowestPrice = scraperAmazon.read(url)
            print "LowestPrice ",lowestPrice
            inStock = 0
            if lowestPrice > 0:
                inStock = 1
            if lowestPrice > 0:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':inStock,'dealType':data['dealType']}}, multi=True)
            else:
                lowestPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':0,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'dealType':data['dealType']}}, multi=True)
            
            print "Returning info"
            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':inStock,'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
        except:
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
        
    elif source_id ==4:
        
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            try:
                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id'] 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
            except:
                pass
            
            url = "http://109.74.200.220:8080/mobileapi/dtr-pricing?id=%s"%(data['identifier'])
            lowestPrice = 0.0
            instock = 0
            req = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(req)
            json_input = response.read()
            response.close()
            priceInfo = json.loads(json_input)
            lowestPrice = priceInfo['response']['sellingPrice']
            if lowestPrice > 0:
                instock = 1
            if instock  == 1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':instock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestPrice , 'in_stock':instock}}, multi=True)
            else:
                lowestPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':instock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':instock}}, multi=True)
            
            print "Returning info"    
            return {'_id':data['_id'],'available_price':lowestPrice,'in_stock':instock,'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
    
        except:
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':4,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
        
        
    elif source_id ==3:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            try:
                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id']
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
                
            except Exception as e:
                print "Exception snapdeal"
                print e
                pass
            
            
            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(data['identifier'])
            req = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(req)
            json_input = response.read()
            response.close()
            vendorInfo = json.loads(json_input)
            lowestOfferPrice = 0
            inStock = 0
            for vendor in vendorInfo:
                lowestOfferPrice = float(vendor['sellingPrice'])
                stock = vendor['buyableInventory']
                if stock > 0 and lowestOfferPrice > 0:
                    inStock = 1
                    break
                    
            print lowestOfferPrice
            print inStock
            print "*************"
            if inStock  == 1:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestOfferPrice , 'in_stock':inStock}}, multi=True)
            else:
                lowestOfferPrice = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
            
            print "Returning info"
            return {'_id':data['_id'],'available_price':lowestOfferPrice,'in_stock':inStock,'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
        except:
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':3,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
        
    elif source_id == 2:
        try:
            if data['identifier'] is None or len(data['identifier'].strip())==0:
                return {}
            
            try:
                if data['priceUpdatedOn'] > to_java_date(now - timedelta(minutes=5)):
                    print "sku id is already updated",data['_id']
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']} 
            except:
                pass
            
            lowestSp = 0
            inStock = 0
            scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
            try:
                if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
                    result = scraperProductPage.read(data['marketPlaceUrl'])
                    print result
                    if result.get('lowestSp')!=0:
                        lowestSp = result.get('lowestSp')
                        inStock = result.get('inStock')
            except:
                print "Unable to scrape product page ",data['identifier']
            if lowestSp ==0:
                url = "http://www.flipkart.com/ps/%s"%(data['identifier'])
                scraperFk = FlipkartScraper.FlipkartScraper()
                vendorsData = scraperFk.read(url)
                sortedVendorsData = []
                sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
                print "data",sortedVendorsData
                for vData in sortedVendorsData:
                    lowestSp = vData['sellingPrice']
                    break
                if lowestSp > 0:
                    inStock = 1
            print lowestSp
            print inStock
            if lowestSp > 0:
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
            else:
                lowestSp = data['available_price']
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock,'priceUpdatedOn':to_java_date(now)}}, multi=True)
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
                
            print "returning"    
            return {'_id':data['_id'],'available_price':lowestSp,'in_stock':inStock,'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
            
        except:
            return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
    else:
        return {}


def populateNegativeDeals():
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
    mc.set("negative_deals", negativeDeals, 600)

def recomputeDeal(skuBundleId):
    """Lets recompute deal for this bundle"""
    print "Recomputing for bundleId",skuBundleId
    
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
    bestPrice = float("inf")
    bestOne = None
    bestSellerPoints = 0
    toUpdate = []
    for similarItem in similarItems:
        if mc.get("negative_deals") is None:
            populateNegativeDeals()
        if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
            continue
        if similarItem['available_price'] < bestPrice:
            bestOne = similarItem
            bestPrice = similarItem['available_price']
            bestSellerPoints = similarItem['bestSellerPoints']
        elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
            bestOne = similarItem
            bestPrice = similarItem['available_price']
            bestSellerPoints = similarItem['bestSellerPoints']
        else:
            pass
    if bestOne is not None:
        for similarItem in similarItems:
            toUpdate.append(similarItem['_id'])
        toUpdate.remove(bestOne['_id'])
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
    if len(toUpdate) > 0:
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
    
    print "Done with recomputing"

def getLatestPrice(skuBundleId, source_id):
    temp = []
    itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':skuBundleId,'source_id' : source_id}))
    for item in itemIds:
        item['dealFlag'] = 0
        item['dealType'] = 0
        manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':source_id, 'sku':item['_id']}))
        if len(manualDeals) > 0:
            item['dealFlag'] = 1
            item['dealType'] =manualDeals[0]['dealType'] 
        temp.append(returnLatestPrice(item, source_id))
    try:
        thread = threading.Thread(target=recomputeDeal, args = (skuBundleId,))
        thread.daemon = True
        thread.start()    
    except:
        print "Unable to compute deal for ",skuBundleId
    return temp

def getLatestPriceById(id):
    item = list(get_mongo_connection().Catalog.MasterData.find({'_id':id}))
    item[0]['dealFlag'] = 0
    item[0]['dealType'] = 0
    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item[0]['source_id'], 'sku':item[0]['_id']}))
    if len(manualDeals) > 0:
        item[0]['dealFlag'] = 1
        item[0]['dealType'] =manualDeals[0]['dealType'] 
    
    info = returnLatestPrice(item[0], item[0]['source_id'])
    try:
        thread = threading.Thread(target=recomputeDeal, args = (item[0]['skuBundleId'],))
        thread.daemon = True
        thread.start()    
    except:
        print "Unable to compute deal for ",item[0]['skuBundleId']
    return info


def main():
    print "retuned %s"%(str(getLatestPrice(17, 2)))

if __name__=='__main__':
    main()