Subversion Repositories SmartDukaan

Rev

Rev 15895 | Rev 16021 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 15895 Rev 16019
Line 1... Line 1...
1 
import urllib2
 1 
import urllib2
2 
from BeautifulSoup import BeautifulSoup
 2 
from BeautifulSoup import BeautifulSoup
3 
import pymongo
 3 
import pymongo
4 
import re
 4 
import re
5 
from dtr.utils.utils import to_java_date
 5 
from dtr.utils.utils import to_java_date, getNlcPoints
6 
import optparse
 6 
import optparse
7 
from datetime import datetime
 7 
from datetime import datetime
8 
import smtplib
 8 
import smtplib
9 
from email.mime.text import MIMEText
 9 
from email.mime.text import MIMEText
10 
from email.mime.multipart import MIMEMultipart
 10 
from email.mime.multipart import MIMEMultipart
11 
from dtr.utils import ShopCluesScraper
 11 
from dtr.utils import ShopCluesScraper
12 
import traceback
 12 
import traceback
- 
 
 13 
from dtr.storage.MemCache import MemCache
13 
 
 14 
 
14 
con = None
 15 
con = None
15 
parser = optparse.OptionParser()
 16 
parser = optparse.OptionParser()
16 
parser.add_option("-m", "--m", dest="mongoHost",
 17 
parser.add_option("-m", "--m", dest="mongoHost",
17 
                      default="localhost",
 18 
                      default="localhost",
18 
                      type="string", help="The HOST where the mongo server is running",
 19 
                      type="string", help="The HOST where the mongo server is running",
19 
                      metavar="mongo_host")
 20 
                      metavar="mongo_host")
20 
 
 21 
 
21 
(options, args) = parser.parse_args()
 22 
(options, args) = parser.parse_args()
22 
 
 23 
 
23 
 
 - 
 
- 
 
 24 
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
24 
exceptionList = []
 25 
exceptionList = []
25 
bestSellers = []
 26 
bestSellers = []
26 
baseUrl = "http://m.shopclues.com/products/getProductList/mobiles:top-selling-mobiles-and-tablets.html/%s/page=%s"
 27 
baseUrl = "http://m.shopclues.com/products/getProductList/mobiles:top-selling-mobiles-and-tablets.html/%s/page=%s"
27 
headers = {
 28 
headers = {
28 
            'User-Agent':'Mozilla/5.0 (Linux; Android 4.3; Nexus 7 Build/JSS15Q) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Safari/537.36',
 29 
            'User-Agent':'Mozilla/5.0 (Linux; Android 4.3; Nexus 7 Build/JSS15Q) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Safari/537.36',
29 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
30 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
30 
            'Accept-Language' : 'en-US,en;q=0.8',
31 
            'Accept-Language' : 'en-US,en;q=0.8',
31 
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
 32 
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
32 
            'Connection':'keep-alive',
 33 
            'Connection':'keep-alive'
33 
            'Accept-Encoding' : 'gzip,deflate,sdch'
 - 
 
34 
        }
 34 
        }
35 
now = datetime.now()
 35 
now = datetime.now()
- 
 
 36 
mc = MemCache(options.mongoHost)
36 
sc = ShopCluesScraper.ShopCluesScraper()
 37 
sc = ShopCluesScraper.ShopCluesScraper()
37 
 
 38 
 
38 
 
 39 
 
39 
class __ProductInfo:
 40 
class __ProductInfo:
40
41
Line 66... Line 67...
66 
    global RETRY_COUNT
 67 
    global RETRY_COUNT
67 
    RETRY_COUNT = 1
68 
    RETRY_COUNT = 1
68 
    while RETRY_COUNT < 10:
 69 
    while RETRY_COUNT < 10:
69 
        try:
 70 
        try:
70 
            soup = None
 71 
            soup = None
71 
            request = urllib2.Request(url, headers)
 72 
            request = urllib2.Request(url, headers=headers)
72 
            response = urllib2.urlopen(request)
73 
            response = urllib2.urlopen(request)
73 
            response_data = response.read()
 74 
            response_data = response.read()
74 
            response.close()
 75 
            response.close()
75 
            try:
 76 
            try:
76 
                page=response_data.decode("utf-8")
 77 
                page=response_data.decode("utf-8")
77 
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
 78 
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
78 
            except:
 79 
            except:
- 
 
 80 
                print traceback.print_exc()
79 
                soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
 81 
                soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
80 
            if soup is None:
 82 
            if soup is None:
81 
                raise
 83 
                raise
82 
            return soup
 84 
            return soup
83 
        except Exception as e:
 85 
        except Exception as e:
84 
            print e
 86 
            traceback.print_exc()
85 
            print "Retrying"
 87 
            print "Retrying"
86 
            RETRY_COUNT = RETRY_COUNT + 1
 88 
            RETRY_COUNT = RETRY_COUNT + 1
87 
 
 89 
 
88
90
89 
def scrapeBestSellers():
 91 
def scrapeBestSellers():
Line 110... Line 112...
110 
            except Exception as e:
 112 
            except Exception as e:
111 
                traceback.print_exc()
 113 
                traceback.print_exc()
112 
                continue
 114 
                continue
113 
            product = list(get_mongo_connection().Catalog.MasterData.find({'source_id':5,'identifier':scin}))
 115 
            product = list(get_mongo_connection().Catalog.MasterData.find({'source_id':5,'identifier':scin}))
114 
            if len(product) > 0:
 116 
            if len(product) > 0:
- 
 
 117 
                if productInfo['inStock'] ==1:
115 
                get_mongo_connection().Catalog.MasterData.update({'source_id':5,'identifier':scin},{"$set":{'rank':rank, 'available_price':productInfo['price'], \
 118 
                    get_mongo_connection().Catalog.MasterData.update({'_id':product[0]['_id']},{"$set":{'rank':rank, 'available_price':productInfo['price'], \
116 
                                                                                                            'in_stock':productInfo['inStock'], 'codAvailable':productInfo['isCod'], \
 119 
                                                                                                            'in_stock':productInfo['inStock'], 'codAvailable':productInfo['isCod'], \
117 
                                                                                                            'coupon':productInfo['coupon']}})
 120 
                                                                                                            'coupon':productInfo['coupon']}, 'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now())})
- 
 
 121 
                    get_mongo_connection().Catalog.Deals.update({'_id':product[0]['_id']}, {'$set' : {'available_price':productInfo['price'] , 'in_stock':productInfo['inStock'],'codAvailable':productInfo['isCod']}})
- 
 
 122 
                else:
- 
 
 123 
                    get_mongo_connection().Catalog.MasterData.update({'_id':product[0]['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now())}})
- 
 
 124 
                    get_mongo_connection().Catalog.Deals.update({'_id':product[0]['_id']}, {'$set' : {'in_stock':0}})
- 
 
 125
- 
 
 126 
                try:
- 
 
 127 
                    recomputeDeal(product[0])
- 
 
 128 
                except:
- 
 
 129 
                    print "Unable to compute deal for %s"%(product[0]['skuBundleId'])
- 
 
 130
118 
            else:
 131 
            else:
119 
                #Lets bundle product by finding similar url pattern
 132 
                #Lets bundle product by finding similar url pattern
120 
                uri = link.replace('http://m.shopclues.com','').replace(".html","")
 133 
                uri = link.replace('http://m.shopclues.com','').replace(".html","")
121 
                try:
 134 
                try:
122 
                    int(uri[uri.rfind('-')+1:])
 135 
                    int(uri[uri.rfind('-')+1:])
Line 128... Line 141...
128 
                if len(product) > 0:
 141 
                if len(product) > 0:
129 
                    bundleNewProduct(product[0], toBundle)
 142 
                    bundleNewProduct(product[0], toBundle)
130 
                else:
 143 
                else:
131 
                    exceptionList.append(toBundle)
 144 
                    exceptionList.append(toBundle)
132 
        page = page+1
 145 
        page = page+1
- 
 
 146 
 
- 
 
 147 
def populateNegativeDeals():
- 
 
 148 
    negativeDeals = get_mongo_connection().Catalog.NegativeDeals.find().distinct('sku')
- 
 
 149 
    mc.set("negative_deals", negativeDeals, 600)
- 
 
 150 
 
- 
 
 151 
def recomputePoints(item, deal):
- 
 
 152 
    try:
- 
 
 153 
        nlcPoints = getNlcPoints(item, deal['minNlc'], deal['maxNlc'], deal['available_price'])
- 
 
 154 
    except:
- 
 
 155 
        traceback.print_exc()
- 
 
 156 
        nlcPoints = deal['nlcPoints']
- 
 
 157 
    if item['manualDealThresholdPrice'] >= deal['available_price']:
- 
 
 158 
        dealPoints = item['dealPoints']
- 
 
 159 
    else:
- 
 
 160 
        dealPoints = 0
- 
 
 161 
    get_mongo_connection().Catalog.Deals.update({'_id':deal['_id']},{"$set":{'totalPoints':deal['totalPoints'] - deal['nlcPoints'] + nlcPoints - deal['dealPoints'] +dealPoints , 'nlcPoints': nlcPoints, 'dealPoints': dealPoints, 'manualDealThresholdPrice': item['manualDealThresholdPrice']}})
- 
 
 162 
 
- 
 
 163
- 
 
 164 
def recomputeDeal(item):
- 
 
 165 
    """Lets recompute deal for this bundle"""
- 
 
 166 
    print "Recomputing for bundleId %d" %(item.get('skuBundleId'))
- 
 
 167 
    skuBundleId = item['skuBundleId']
- 
 
 168
- 
 
 169 
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
- 
 
 170 
    bestPrice = float("inf")
- 
 
 171 
    bestOne = None
- 
 
 172 
    bestSellerPoints = 0
- 
 
 173 
    toUpdate = []
- 
 
 174 
    prepaidBestPrice = float("inf")
- 
 
 175 
    prepaidBestOne = None
- 
 
 176 
    prepaidBestSellerPoints = 0
- 
 
 177 
    for similarItem in similarItems:
- 
 
 178 
        if similarItem['_id'] == item['_id']:
- 
 
 179 
            try:
- 
 
 180 
                recomputePoints(item, similarItem)
- 
 
 181 
            except:
- 
 
 182 
                traceback.print_exc()
- 
 
 183 
        if similarItem['codAvailable'] ==1:
- 
 
 184 
            if mc.get("negative_deals") is None:
- 
 
 185 
                populateNegativeDeals()
- 
 
 186 
            if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
- 
 
 187 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
- 
 
 188 
                continue
- 
 
 189 
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
- 
 
 190 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
- 
 
 191 
                continue
- 
 
 192 
            if similarItem['available_price'] < bestPrice:
- 
 
 193 
                bestOne = similarItem
- 
 
 194 
                bestPrice = similarItem['available_price']
- 
 
 195 
                bestSellerPoints = similarItem['bestSellerPoints']
- 
 
 196 
            elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
- 
 
 197 
                bestOne = similarItem
- 
 
 198 
                bestPrice = similarItem['available_price']
- 
 
 199 
                bestSellerPoints = similarItem['bestSellerPoints']
- 
 
 200 
            else:
- 
 
 201 
                pass
- 
 
 202 
        else:
- 
 
 203 
            if mc.get("negative_deals") is None:
- 
 
 204 
                populateNegativeDeals()
- 
 
 205 
            if similarItem['in_stock'] == 0 or similarItem['maxprice'] is None or similarItem['maxprice'] < similarItem['available_price'] or similarItem['_id'] in mc.get("negative_deals"):
- 
 
 206 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0, 'prepaidDeal':0 }})
- 
 
 207 
                continue
- 
 
 208 
            if similarItem['source_id'] == SOURCE_MAP.get('SHOPCLUES.COM') and similarItem['rank']==0:
- 
 
 209 
                get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':0 }})
- 
 
 210 
                continue
- 
 
 211 
            if similarItem['available_price'] < prepaidBestPrice:
- 
 
 212 
                prepaidBestOne = similarItem
- 
 
 213 
                prepaidBestPrice = similarItem['available_price']
- 
 
 214 
                prepaidBestSellerPoints = similarItem['bestSellerPoints']
- 
 
 215 
            elif similarItem['available_price'] == prepaidBestPrice and prepaidBestSellerPoints < similarItem['bestSellerPoints']:
- 
 
 216 
                prepaidBestOne = similarItem
- 
 
 217 
                prepaidBestPrice = similarItem['available_price']
- 
 
 218 
                prepaidBestSellerPoints = similarItem['bestSellerPoints']
- 
 
 219 
            else:
- 
 
 220 
                pass
- 
 
 221 
    if bestOne is not None and prepaidBestOne is not None:
- 
 
 222 
        for similarItem in similarItems:
- 
 
 223 
            toUpdate.append(similarItem['_id'])
- 
 
 224 
        toUpdate.remove(bestOne['_id'])
- 
 
 225 
        toUpdate.remove(prepaidBestOne['_id'])
- 
 
 226 
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1,'prepaidDeal':0 }})
- 
 
 227 
        get_mongo_connection().Catalog.Deals.update({ '_id' : prepaidBestOne['_id'] }, {'$set':{'showDeal':0,'prepaidDeal':1 }})
- 
 
 228 
    if len(toUpdate) > 0:
- 
 
 229 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
- 
 
 230 
 
133
231
134 
def bundleNewProduct(existingProduct, toBundle):
 232 
def bundleNewProduct(existingProduct, toBundle):
135 
    print "Adding new product"
 233 
    print "Adding new product"
136 
    try:
 234 
    try:
137 
        max_id = list(get_mongo_connection().Catalog.MasterData.find().sort([('_id',pymongo.DESCENDING)]).limit(1))
 235 
        max_id = list(get_mongo_connection().Catalog.MasterData.find().sort([('_id',pymongo.DESCENDING)]).limit(1))