Subversion Repositories SmartDukaan

Rev

Rev 16084 | Rev 16099 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 16084 Rev 16095
Line 20... Line 20...
20 
                      metavar="mongo_host")
 20 
                      metavar="mongo_host")
21 
 
 21 
 
22 
(options, args) = parser.parse_args()
 22 
(options, args) = parser.parse_args()
23 
 
 23 
 
24 
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
 24 
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3,'SAHOLIC':4, 'SHOPCLUES.COM':5}
25 
exceptionList = []
 - 
 
26 
bestSellers = []
 25 
bestSellers = []
27 
baseUrl = "http://m.shopclues.com/products/getProductList/mobiles:top-selling-mobiles-and-tablets.html/%s/page=%s"
 26 
baseUrl = "http://m.shopclues.com/products/getProductList/mobiles:top-selling-mobiles-and-tablets.html/%s/page=%s"
28 
headers = {
 27 
headers = {
29 
            'User-Agent':'Mozilla/5.0 (Linux; Android 4.3; Nexus 7 Build/JSS15Q) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Safari/537.36',
 28 
            'User-Agent':'Mozilla/5.0 (Linux; Android 4.3; Nexus 7 Build/JSS15Q) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Safari/537.36',
30 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
29 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
Line 34... Line 33...
34 
        }
 33 
        }
35 
now = datetime.now()
 34 
now = datetime.now()
36 
mc = MemCache(options.mongoHost)
 35 
mc = MemCache(options.mongoHost)
37 
sc = ShopCluesScraper.ShopCluesScraper()
 36 
sc = ShopCluesScraper.ShopCluesScraper()
38 
 
 37 
 
- 
 
 38 
bundledProducts = []
- 
 
 39 
exceptionList = []
- 
 
 40 
 
39 
 
 41 
 
40 
class __ProductInfo:
 42 
class __ProductInfo:
41
43
42 
    def __init__(self, identifier, rank, url, available_price, in_stock, codAvailable, source_product_name, thumbnail, coupon):
 44 
    def __init__(self, identifier, rank, url, available_price, in_stock, codAvailable, source_product_name, thumbnail, coupon):
43 
        self.identifier = identifier
 45 
        self.identifier = identifier
Line 46... Line 48...
46 
        self.available_price = available_price
 48 
        self.available_price = available_price
47 
        self.in_stock = in_stock
 49 
        self.in_stock = in_stock
48 
        self.codAvailable = codAvailable
 50 
        self.codAvailable = codAvailable
49 
        self.source_product_name = source_product_name
 51 
        self.source_product_name = source_product_name
50 
        self.thumbnail = thumbnail
 52 
        self.thumbnail = thumbnail
51 
        self.coupon = coupon
53 
        self.coupon = coupon
- 
 
 54 
 
- 
 
 55 
class __NewBundled:
- 
 
 56 
    def __init__(self, newProduct, oldProduct):
- 
 
 57 
        self.newProduct = newProduct
- 
 
 58 
        self.oldProduct = oldProduct
52 
 
 59 
 
53 
def get_mongo_connection(host=options.mongoHost, port=27017):
 60 
def get_mongo_connection(host=options.mongoHost, port=27017):
54 
    global con
 61 
    global con
55 
    if con is None:
 62 
    if con is None:
56 
        print "Establishing connection %s host and port %d" %(host,port)
 63 
        print "Establishing connection %s host and port %d" %(host,port)
Line 88... Line 95...
88 
            RETRY_COUNT = RETRY_COUNT + 1
 95 
            RETRY_COUNT = RETRY_COUNT + 1
89 
 
 96 
 
90
97
91 
def scrapeBestSellers():
 98 
def scrapeBestSellers():
92 
    global bestSellers
 99 
    global bestSellers
- 
 
 100 
    global exceptionList
93 
    bestSellers = []
 101 
    bestSellers = []
94 
    rank = 0
 102 
    rank = 0
95 
    page = 1
 103 
    page = 1
96 
    while (True):
 104 
    while (True):
97 
        url = (baseUrl)%(page,page-1)
 105 
        url = (baseUrl)%(page,page-1)
Line 248... Line 256...
248 
    if len(toUpdate) > 0:
 256 
    if len(toUpdate) > 0:
249 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
 257 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0,'prepaidDeal':0 }},upsert=False, multi=True)
250 
 
 258 
 
251
259
252 
def bundleNewProduct(existingProduct, toBundle):
 260 
def bundleNewProduct(existingProduct, toBundle):
- 
 
 261 
    global bundledProducts
- 
 
 262 
    global exceptionList
253 
    print "Adding new product"
 263 
    print "Adding new product"
254 
    try:
 264 
    try:
255 
        max_id = list(get_mongo_connection().Catalog.MasterData.find().sort([('_id',pymongo.DESCENDING)]).limit(1))
 265 
        max_id = list(get_mongo_connection().Catalog.MasterData.find().sort([('_id',pymongo.DESCENDING)]).limit(1))
256 
        existingProduct['_id'] = max_id[0]['_id'] + 1
 266 
        existingProduct['_id'] = max_id[0]['_id'] + 1
257 
        existingProduct['addedOn'] = to_java_date(datetime.now())
 267 
        existingProduct['addedOn'] = to_java_date(datetime.now())
258 
        existingProduct['available_price'] = toBundle.available_price
 268 
        existingProduct['available_price'] = toBundle.available_price
259 
        existingProduct['updatedOn'] = to_java_date(datetime.now())
 269 
        existingProduct['updatedOn'] = to_java_date(datetime.now())
260 
        existingProduct['codAvailable'] = toBundle.codAvailable
 270 
        existingProduct['codAvailable'] = toBundle.codAvailable
261 
        existingProduct['coupon'] = toBundle.coupon
 271 
        existingProduct['coupon'] = str(toBundle.coupon)
262 
        existingProduct['identifier'] = toBundle.identifier
 272 
        existingProduct['identifier'] = str(toBundle.identifier)
263 
        existingProduct['in_stock'] = toBundle.in_stock
 273 
        existingProduct['in_stock'] = toBundle.in_stock
264 
        existingProduct['marketPlaceUrl'] = toBundle.url
 274 
        existingProduct['marketPlaceUrl'] = toBundle.url
265 
        existingProduct['rank'] = toBundle.rank
 275 
        existingProduct['rank'] = toBundle.rank
266 
        existingProduct['source_product_name'] = toBundle.source_product_name
 276 
        existingProduct['source_product_name'] = toBundle.source_product_name
267 
        existingProduct['url'] = toBundle.url
 277 
        existingProduct['url'] = toBundle.url
268 
        get_mongo_connection().Catalog.MasterData.insert(existingProduct)
 278 
        get_mongo_connection().Catalog.MasterData.insert(existingProduct)
- 
 
 279 
        newBundled = __NewBundled(toBundle, existingProduct)
- 
 
 280 
        bundledProducts.append(newBundled)
269 
        return {1:'Data added successfully.'}
 281 
        return {1:'Data added successfully.'}
270 
    except Exception as e:
 282 
    except Exception as e:
271 
        print e
 283 
        print e
- 
 
 284 
        exceptionList.append(toBundle)
272 
        return {0:'Unable to add data.'}
 285 
        return {0:'Unable to add data.'}
273 
 
 286 
 
274 
def exceptionItems():
 287 
def sendMail():
- 
 
 288 
    message="""<html>
- 
 
 289 
            <body>
- 
 
 290 
            <h3>ShopClues Best Sellers Auto Bundled</h3>
- 
 
 291 
            <table border="1" style="width:100%;">
- 
 
 292 
            <thead>
- 
 
 293 
            <tr>
- 
 
 294 
            <th>Item Id</th>
- 
 
 295 
            <th>Identifier</th>
- 
 
 296 
            <th>Rank</th>
- 
 
 297 
            <th>Product Name</th>
- 
 
 298 
            <th>Bundle Id</th>
- 
 
 299 
            <th>Bundled with Brand</th>
- 
 
 300 
            <th>Bundled with Product Name</th>
- 
 
 301 
            <th>Available_price</th>
- 
 
 302 
            <th>In Stock</th>
- 
 
 303 
            <th>Coupon</th>
- 
 
 304 
            <th>COD Available</th>
- 
 
 305 
            </tr></thead>
- 
 
 306 
            <tbody>"""
- 
 
 307 
    for bundledProduct in bundledProducts:
- 
 
 308 
        newProduct = bundledProduct.newProduct
- 
 
 309 
        oldProduct = bundledProduct.oldProduct
- 
 
 310 
        message+="""<tr>
- 
 
 311 
        <td style="text-align:center">"""+str(oldProduct.get('_id'))+"""</td>
- 
 
 312 
        <td style="text-align:center">"""+oldProduct.get('identifier')+"""</td>
- 
 
 313 
        <td style="text-align:center">"""+str(oldProduct.get('rank'))+"""</td>
- 
 
 314 
        <td style="text-align:center">"""+(oldProduct.get('source_product_name'))+"""</td>
- 
 
 315 
        <td style="text-align:center">"""+str(oldProduct.get('skuBundleId'))+"""</td>
- 
 
 316 
        <td style="text-align:center">"""+(oldProduct.get('brand'))+"""</td>
- 
 
 317 
        <td style="text-align:center">"""+(oldProduct.get('product_name'))+"""</td>
- 
 
 318 
        <td style="text-align:center">"""+str(oldProduct.get('available_price'))+"""</td>
- 
 
 319 
        <td style="text-align:center">"""+str(oldProduct.get('in_stock'))+"""</td>
- 
 
 320 
        <td style="text-align:center">"""+str(oldProduct.get('coupon'))+"""</td>
- 
 
 321 
        <td style="text-align:center">"""+str(oldProduct.get('codAvailable'))+"""</td>
- 
 
 322 
        </tr>"""
- 
 
 323 
    message+="""</tbody></table><h3>Items not bundled</h3><table border="1" style="width:100%;">
- 
 
 324 
    <tr>
- 
 
 325 
    <th>Identifier</th>
- 
 
 326 
    <th>Rank</th>
- 
 
 327 
    <th>Product Name</th>
- 
 
 328 
    <th>Url</th>
- 
 
 329 
    <th>Available Price</th>
- 
 
 330 
    <th>In Stock</th>
- 
 
 331 
    <th>COD Available</th>
- 
 
 332 
    <th>Coupon</th>
- 
 
 333 
    </tr></thead>
- 
 
 334 
    <tbody>"""
275 
    for item in exceptionList:
 335 
    for exceptionItem in exceptionList:
- 
 
 336 
        message+="""<tr>
- 
 
 337 
        <td style="text-align:center">"""+str(exceptionItem.identifier)+"""</td>
- 
 
 338 
        <td style="text-align:center">"""+str(exceptionItem.rank)+"""</td>
- 
 
 339 
        <td style="text-align:center">"""+(exceptionItem.source_product_name)+"""</td>
- 
 
 340 
        <td style="text-align:center">"""+(exceptionItem.url)+"""</td>
- 
 
 341 
        <td style="text-align:center">"""+str(exceptionItem.available_price)+"""</td>
- 
 
 342 
        <td style="text-align:center">"""+str(exceptionItem.in_stock)+"""</td>
- 
 
 343 
        <td style="text-align:center">"""+str(exceptionItem.codAvailable)+"""</td>
- 
 
 344 
        <td style="text-align:center">"""+str(exceptionItem.coupon)+"""</td>
- 
 
 345 
        </tr>"""
- 
 
 346 
    message+="""</tbody></table></body></html>"""
276 
        print vars(item)
 347 
    print message
- 
 
 348 
    recipients = ['kshitij.sood@saholic.com']
- 
 
 349 
    #recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
- 
 
 350 
    msg = MIMEMultipart()
- 
 
 351 
    msg['Subject'] = "Shopclues Best Sellers" + ' - ' + str(datetime.now())
- 
 
 352 
    msg['From'] = ""
- 
 
 353 
    msg['To'] = ",".join(recipients)
- 
 
 354 
    msg.preamble = "Shopclues Best Sellers" + ' - ' + str(datetime.now())
- 
 
 355 
    html_msg = MIMEText(message, 'html')
- 
 
 356 
    msg.attach(html_msg)
- 
 
 357
- 
 
 358 
    smtpServer = smtplib.SMTP('localhost')
- 
 
 359 
    smtpServer.set_debuglevel(1)
- 
 
 360 
    sender = 'dtr@shop2020.in'
- 
 
 361 
    try:
- 
 
 362 
        smtpServer.sendmail(sender, recipients, msg.as_string())
- 
 
 363 
        print "Successfully sent email"
- 
 
 364 
    except:
- 
 
 365 
        print "Error: unable to send email."
- 
 
 366 
 
- 
 
 367
- 
 
 368
277
369
278 
 
 370 
 
279 
def main():
 371 
def main():
280 
    scrapeBestSellers()
 372 
    scrapeBestSellers()
281 
    exceptionItems()
 373 
    sendMail()
282
374
283 
if __name__=='__main__':
 375 
if __name__=='__main__':
284 
    main()
 376 
    main()
285 

            377