Subversion Repositories SmartDukaan

import urllib2

from BeautifulSoup import BeautifulSoup

import pymongo

import re

from dtr.utils.utils import to_java_date, getNlcPoints

import optparse

from datetime import datetime

import smtplib

from email.mime.text import MIMEText

from email.mime.multipart import MIMEMultipart

from dtr.utils import ShopCluesScraper

import traceback

from dtr.storage.MemCache import MemCache

con = None

parser = optparse.OptionParser()

parser.add_option("-m", "--m", dest="mongoHost",

                      default="localhost",

                      type="string", help="The HOST where the mongo server is running",

                      metavar="mongo_host")

(options, args) = parser.parse_args()

SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}

exceptionList = []

bestSellers = []

baseUrl = "http://m.shopclues.com/products/getProductList/mobiles:top-selling-mobiles-and-tablets.html/%s/page=%s"

headers = {

            'User-Agent':'Mozilla/5.0 (Linux; Android 4.3; Nexus 7 Build/JSS15Q) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Safari/537.36',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',

            'Connection':'keep-alive'

        }

now = datetime.now()

mc = MemCache(options.mongoHost)

sc = ShopCluesScraper.ShopCluesScraper()

class __ProductInfo:

    def __init__(self, identifier, rank, url, available_price, in_stock, codAvailable, source_product_name, thumbnail, coupon):

        self.identifier = identifier

        self.rank  = rank

        self.url = url

        self.available_price = available_price

        self.in_stock = in_stock

        self.codAvailable = codAvailable

        self.source_product_name = source_product_name

        self.thumbnail = thumbnail

        self.coupon = coupon 

def get_mongo_connection(host=options.mongoHost, port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def getSoupObject(url):

    print "Getting soup object for"

    print url

    global RETRY_COUNT

    RETRY_COUNT = 1 

    while RETRY_COUNT < 10:

        try:

            soup = None

            request = urllib2.Request(url, headers=headers)

            response = urllib2.urlopen(request)   

            response_data = response.read()

            response.close()

            try:

                page=response_data.decode("utf-8")

                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            except:

                print traceback.print_exc()

                soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)

            if soup is None:

                raise

            return soup

        except Exception as e:

            traceback.print_exc()

            print "Retrying"

            RETRY_COUNT = RETRY_COUNT + 1

def scrapeBestSellers():

    global bestSellers

    bestSellers = []

    rank = 0

    page = 1

    while (True):

        url = (baseUrl)%(page,page-1)

        soup = getSoupObject(url)

        productDivs = soup.findAll('div',{'class':'pd-list-cont'})

        if productDivs is None or len(productDivs)==0:

            return

        for productDiv in productDivs:

            rank = rank + 1

            info_tag =  productDiv.find('a')

            link = info_tag['href']

            scin = info_tag['data-id'].strip()

            print link

            print scin

            productName = productDiv.find('div',{'class':'pdt-name'}).string

            try:

                productInfo = sc.read(link)

            except Exception as e:

                traceback.print_exc()

                continue

            product = list(get_mongo_connection().Catalog.MasterData.find({'source_id':5,'identifier':scin}))

            if len(product) > 0:

                if productInfo['inStock'] ==1:

                    get_mongo_connection().Catalog.MasterData.update({'_id':product[0]['_id']},{"$set":{'rank':rank, 'available_price':productInfo['price'], \

                                                                                                            'in_stock':productInfo['inStock'], 'codAvailable':productInfo['isCod'], \

                                                                                                            'coupon':productInfo['coupon'], 'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now())}})

                    get_mongo_connection().Catalog.Deals.update({'_id':product[0]['_id']}, {'$set' : {'rank':rank,'available_price':productInfo['price'] , 'in_stock':productInfo['inStock'],'codAvailable':productInfo['isCod']}})

                else:

                    get_mongo_connection().Catalog.MasterData.update({'_id':product[0]['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now())}})

                    get_mongo_connection().Catalog.Deals.update({'_id':product[0]['_id']}, {'$set' : {'in_stock':0}})

                try:

                    recomputeDeal(product[0])

                except:

                    print "Unable to compute deal for %s"%(product[0]['skuBundleId'])

            else:

                #Lets bundle product by finding similar url pattern

                uri = link.replace('http://m.shopclues.com','').replace(".html","")

                try:

                    int(uri[uri.rfind('-')+1:])

                    uri =  uri[:uri.rfind('-')]

                except:

                    pass

                product = list(get_mongo_connection().Catalog.MasterData.find({'source_id':5,'marketPlaceUrl':{'$regex': uri}}))

                toBundle = __ProductInfo(scin, rank, link, productInfo['price'], productInfo['inStock'],productInfo['isCod'], productName, "" ,productInfo['coupon'])

                if len(product) > 0:

                    bundleNewProduct(product[0], toBundle)

                    try:

                        recomputeDeal(product[0])

                    except:

                        print "Unable to compute deal for %s"%(product[0]['skuBundleId'])

                else:

                    exceptionList.append(toBundle)

        page = page+1

def populateNegativeDeals():

    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')

    mc.set("negative_deals", negativeDeals, 600)

def recomputePoints(item, deal):

    try:

        nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])

    except:

        traceback.print_exc()

        nlcPoints = deal['nlcPoints']

    if item['manualDealThresholdPrice'] >= deal['available_price']:

        dealPoints = item['dealPoints']

    else:

        dealPoints = 0

    get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})

def recomputeDeal(item):

    """Lets recompute deal for this bundle"""

    print "Recomputing for bundleId %d" %(item.get('skuBundleId'))

    skuBundleId = item['skuBundleId']

    item['dealFlag'] = 0

    item['dealType'] = 0

    item['dealPoints'] = 0

    item['manualDealThresholdPrice'] = None

    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':item['source_id'], 'sku':item['_id']}))

    if len(manualDeals) > 0:

        item['dealFlag'] = 1

        item['dealType'] =manualDeals[0]['dealType']

        item['dealPoints'] = manualDeals[0]['dealPoints']

        item['manualDealThresholdPrice'] = manualDeals[0]['dealThresholdPrice']

    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))

    bestPrice = float("inf")

    bestOne = None

    bestSellerPoints = 0

    toUpdate = []

    prepaidBestPrice = float("inf")

    prepaidBestOne = None

    prepaidBestSellerPoints = 0

    for similarItem in similarItems:

        if similarItem['_id'] == item['_id']:

            try:

                recomputePoints(item, similarItem)

            except:

                traceback.print_exc()

        if similarItem['codAvailable'] ==1:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem['available_price'] < bestPrice:

                bestOne = similarItem

                bestPrice = similarItem['available_price']

                bestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:

                bestOne = similarItem

                bestPrice = similarItem['available_price']

                bestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

        else:

            if mc.get("negative_deals") is None:

                populateNegativeDeals()

            if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})

                continue

            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:

                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})

                continue

            if similarItem['available_price'] < prepaidBestPrice:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem['available_price']

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:

                prepaidBestOne = similarItem

                prepaidBestPrice = similarItem['available_price']

                prepaidBestSellerPoints = similarItem['bestSellerPoints']

            else:

                pass

    if bestOne is not None or prepaidBestOne is not None:

        for similarItem in similarItems:

            toUpdate.append(similarItem['_id'])

        if bestOne is not None:

            toUpdate.remove(bestOne['_id'])

            get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})

        if prepaidBestOne is not None:

            if bestOne is not None:

                if prepaidBestOne['available_price'] < bestOne['available_price']: 

                    toUpdate.remove(prepaidBestOne['_id'])

                    get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

            else:

                toUpdate.remove(prepaidBestOne['_id'])

                get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})

    if len(toUpdate) > 0:

        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)

def bundleNewProduct(existingProduct, toBundle):

    print "Adding new product"

    try:

        max_id = list(get_mongo_connection().Catalog.MasterData.find().sort([('_id',pymongo.DESCENDING)]).limit(1))

        existingProduct['_id'] = max_id[0]['_id'] + 1

        existingProduct['addedOn'] = to_java_date(datetime.now())

        existingProduct['available_price'] = toBundle.available_price

        existingProduct['updatedOn'] = to_java_date(datetime.now())

        existingProduct['codAvailable'] = toBundle.codAvailable

        existingProduct['coupon'] = toBundle.coupon

        existingProduct['identifier'] = toBundle.identifier

        existingProduct['in_stock'] = toBundle.in_stock

        existingProduct['marketPlaceUrl'] = toBundle.url

        existingProduct['rank'] = toBundle.rank

        existingProduct['source_product_name'] = toBundle.source_product_name

        existingProduct['url'] = toBundle.url

        get_mongo_connection().Catalog.MasterData.insert(existingProduct)

        return {1:'Data added successfully.'}

    except Exception as e:

        print e

        return {0:'Unable to add data.'}

def exceptionItems():

    for item in exceptionList:

        print vars(item)

def main():

    scrapeBestSellers()

    exceptionItems()

if __name__=='__main__':

    main()