Subversion Repositories SmartDukaan

from BeautifulSoup import BeautifulSoup

from datetime import datetime

from dtr.utils.utils import to_java_date, fetchResponseUsingProxy

from email.mime.multipart import MIMEMultipart

from email.mime.text import MIMEText

import json

import optparse

import pymongo

import re

import smtplib

import urllib2

from dtr.utils.PaytmOfferScraper import fetchOffers

from shop2020.utils.EmailAttachmentSender import get_attachment_part

from shop2020.utils import EmailAttachmentSender

con = None

parser = optparse.OptionParser()

parser.add_option("-m", "--m", dest="mongoHost",

                      default="localhost",

                      type="string", help="The HOST where the mongo server is running",

                      metavar="mongo_host")

(options, args) = parser.parse_args()

exceptionList = []

bestSellers = []

now = datetime.now()

class __RankInfo:

    def __init__(self, identifier, rank, category, available_price, gross_price, in_stock, coupon, thumbnail, source_product_name, marketPlaceUrl, cod):

        self.identifier = identifier

        self.rank  = rank

        self.category = category

        self.available_price = available_price

        self.gross_price = gross_price

        self.in_stock = in_stock

        self.coupon = coupon

        self.thumbnail = thumbnail

        self.source_product_name = source_product_name

        self.marketPlaceUrl = marketPlaceUrl

        self.cod = cod

def get_mongo_connection(host=options.mongoHost, port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def scrapeBestSellerMobiles():

    global bestSellers

    rank = 0

    mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'

    for i in range(1,5):

        data = fetchResponseUsingProxy(mobileCategoryUrl%(i))

        jsonResponse = json.loads(data)

        for jsonProduct in jsonResponse['grid_layout']:

            rank = rank + 1

            identifier = str(jsonProduct['complex_product_id'])

            r_info = __RankInfo(identifier,rank, None, None,None,None,None,None,None,None,None)

            print rank, identifier

            bestSellers.append(r_info)

def commitBestSellers(category):

    global exceptionList

    print "Rank",

    print '\t',

    print 'Identifier'

    for x in bestSellers:

        print x.rank,

        print '\t',

        print x.identifier,

        print '\t',

        col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6}))

        print "count sku",len(col)

        print '\n'

        if len(col) == 0:

            x.category = category

            exceptionList.append(x)

        else:

            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})

def scrapeBestSellerTablets():

    global bestSellers

    bestSellers = []

    rank = 0

    bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'

    for i in range(1,5):

        data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))

        jsonResponse = json.loads(data)

        for jsonProduct in jsonResponse['grid_layout']:

            rank = rank + 1

            identifier = str(jsonProduct['complex_product_id'])

            r_info = __RankInfo(identifier,rank,  None, None,None,None,None,None,None,None,None)

            print rank, identifier

            bestSellers.append(r_info)

def resetRanks(category_id):

    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)

def sendEmail():

    message="""<html>

            <body>

            <h3>PayTM Best Sellers not in master</h3>

            <table border="1" style="width:100%;">

            <thead>

            <tr><th>Identifier</th>

            <th>Category</th>

            <th>Rank</th>

            <th>Available_price</th>

            <th>Gross_price</th>

            <th>In_stock</th>

            <th>Coupon</th>

            <th>Thumbnail</th>

            <th>Source_product_name</th>

            <th>MarketPlaceUrl</th>

            <th>Cod</th>

            </tr></thead>

            <tbody>"""

    for item in exceptionList:

        try:

            message+="""<tr>

            <td style="text-align:center">"""+(item.identifier)+"""</td>

            <td style="text-align:center">"""+(item.category)+"""</td>

            <td style="text-align:center">"""+str(item.rank)+"""</td>

            <td style="text-align:center">"""+str(item.available_price)+"""</td>

            <td style="text-align:center">"""+str(item.gross_price)+"""</td>

            <td style="text-align:center">"""+str(item.in_stock)+"""</td>

            <td style="text-align:center">"""+str(item.coupon)+"""</td>

            <td style="text-align:center">"""+str(item.thumbnail)+"""</td>

            <td style="text-align:center">"""+str(item.source_product_name)+"""</td>

            <td style="text-align:center">"""+str(item.marketPlaceUrl)+"""</td>

            <td style="text-align:center">"""+str(item.cod)+"""</td>

            </tr>"""

        except:

            continue

    message+="""</tbody></table></body></html>"""

    print message

    #recipients = ['amit.gupta@saholic.com']

    recipients = ['kshitij.sood@saholic.com','ritesh.chauhan@saholic.com','aishwarya.singh@saholic.com']

    EmailAttachmentSender.mail_send_grid("dtr@smartdukaan.com","apikey", "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", recipients, "Paytm Best Sellers",message ,[],[],[])              

def addExtraAttributes():

    for e in exceptionList:

        url = "https://catalog.paytm.com/v1/mobile/product/%s" %(e.identifier.strip())

        try:

            response_data = fetchResponseUsingProxy(url,  proxy=False)

        except:

            continue

        input_json = json.loads(response_data)

        offerPrice = float(input_json['offer_price'])

        e.cod = int(input_json['pay_type_supported'].get('COD'))

        offerUrl = (input_json['offer_url'])

        e.in_stock = (input_json['instock'])

        try:

            offers = fetchOffers(offerUrl)

        except:

            continue

        bestOffer = {}

        e.gross_price = float(offerPrice)

        effective_price = offerPrice

        coupon = ""

        for offer_data in offers.get('codes'):

            if effective_price > offer_data.get('effective_price'):

                effective_price = offer_data.get('effective_price')

                bestOffer = offer_data

                coupon = bestOffer.get('code')

        e.source_product_name = input_json['name']

        e.thumbnail = input_json['thumbnail']

        e.marketPlaceUrl = input_json['shareurl']

        e.coupon = coupon

        e.available_price = float(effective_price)

def main():

    scrapeBestSellerMobiles()

    if len(bestSellers) > 0:

        resetRanks(3)

        commitBestSellers("MOBILE")

    scrapeBestSellerTablets()

    if len(bestSellers) > 0:

        resetRanks(5)

        commitBestSellers("TABLET")

    addExtraAttributes()

    sendEmail()

if __name__=='__main__':

    main()