Subversion Repositories SmartDukaan

import urllib2
from BeautifulSoup import BeautifulSoup
import pymongo
import re
from dtr.utils.utils import to_java_date
import optparse
from datetime import datetime
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart

con = None
parser = optparse.OptionParser()
parser.add_option("-m", "--m", dest="mongoHost",
                      default="localhost",
                      type="string", help="The HOST where the mongo server is running",
                      metavar="mongo_host")

(options, args) = parser.parse_args()


exceptionList = []
asin_regex = r'/([A-Z0-9]{10})'
bestSellers = []
now = datetime.now()


class __RankInfo:
    
    def __init__(self, identifier, rank, category):
        self.identifier = identifier
        self.rank  = rank
        self.category = category

def get_mongo_connection(host=options.mongoHost, port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def getSoupObject(url):
    print "Getting soup object for"
    print url
    global RETRY_COUNT
    RETRY_COUNT = 1 
    while RETRY_COUNT < 10:
        try:
            soup = None
            request = urllib2.Request(url)
            request.add_header('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
            request.add_header('Accept-Language','en-US,en;q=0.8,hi;q=0.6')
            request.add_header('Connection','keep-alive')
            request.add_header('User-Agent','Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36')
        
            response = urllib2.urlopen(request)   
            response_data = response.read()
            response.close()
            try:
                page=response_data.decode("utf-8")
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            except:
                soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
            if soup is None:
                raise
            return soup
        except Exception as e:
            print e
            print "Retrying"
            RETRY_COUNT = RETRY_COUNT + 1


def scrapeBestSellerMobiles():
    global bestSellers
    rank = 0
    for i in range(1,6):
        aboveFoldUrl = "http://www.amazon.in/gp/bestsellers/electronics/1389432031/ref=zg_bs_1389432031_pg_%d?ie=UTF8&pg=%d&ajax=1&isAboveTheFold=1" %(i,i) 
        belowFoldUrl = "http://www.amazon.in/gp/bestsellers/electronics/1389432031/ref=zg_bs_1389432031_pg_%d?ie=UTF8&pg=%d&ajax=1&isAboveTheFold=0" %(i,i)
        above_soup = getSoupObject(aboveFoldUrl)
        below_soup = getSoupObject(belowFoldUrl)
        for x in above_soup.findAll('div',{'class':'zg_itemImmersion'}):
            am_url =  x.find('div',{'class':'zg_title'}).find('a')['href']
            identifier =  (re.search(asin_regex, am_url)).group(1)
            rank = rank + 1
            r_info = __RankInfo(identifier,rank, None)
            bestSellers.append(r_info)
        for x in below_soup.findAll('div',{'class':'zg_itemImmersion'}):
            am_url =  x.find('div',{'class':'zg_title'}).find('a')['href']
            identifier =  (re.search(asin_regex, am_url)).group(1)
            rank = rank + 1
            r_info = __RankInfo(identifier,rank, None)
            bestSellers.append(r_info)

def commitBestSellers(category):
    global exceptionList
    print "Rank",
    print '\t',
    print 'Identifier'
    for x in bestSellers:
        print x.rank,
        print '\t',
        print x.identifier,
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier.strip()})
        print "count sku",
        print '\t',
        if len(list(col)) == 0:
            x.category = category
            exceptionList.append(x)
        else:
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier.strip() }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
        
def scrapeBestSellerTablets():
    global bestSellers
    bestSellers = []
    rank = 0
    for i in range(1,6):
        aboveFoldUrl = "http://www.amazon.in/gp/bestsellers/electronics/1375458031/ref=zg_bs_1375458031_pg_%d?ie=UTF8&pg=%d&ajax=1" %(i,i) 
        belowFoldUrl = "http://www.amazon.in/gp/bestsellers/electronics/1375458031/ref=zg_bs_1375458031_pg_%d?ie=UTF8&pg=%d&ajax=1&isAboveTheFold=0" %(i,i)
        above_soup = getSoupObject(aboveFoldUrl)
        below_soup = getSoupObject(belowFoldUrl)
        for x in above_soup.findAll('div',{'class':'zg_itemImmersion'}):
            am_url =  x.find('div',{'class':'zg_title'}).find('a')['href']
            identifier =  (re.search(asin_regex, am_url)).group(1)
            rank = rank + 1
            print identifier,
            print '\t',
            print rank
            r_info = __RankInfo(identifier,rank, None)
            bestSellers.append(r_info)
        for x in below_soup.findAll('div',{'class':'zg_itemImmersion'}):
            am_url =  x.find('div',{'class':'zg_title'}).find('a')['href']
            identifier =  (re.search(asin_regex, am_url)).group(1)
            rank = rank + 1
            print identifier,
            print '\t',
            print rank
            r_info = __RankInfo(identifier,rank, None)
            bestSellers.append(r_info)

def resetRanks(category_id):
    oldRankedItems = get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':1,'category_id':category_id})
    for item in oldRankedItems:
        get_mongo_connection().Catalog.MasterData.update({'_id':item['_id']}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)

def sendEmail():
    message="""<html>
            <body>
            <h3>Amazon Best Sellers not in master</h3>
            <table border="1" style="width:100%;">
            <thead>
            <tr><th>Identifier</th>
            <th>Category</th>
            <th>Rank</th>
            </tr></thead>
            <tbody>"""
    for item in exceptionList:
        message+="""<tr>
        <td style="text-align:center">"""+(item.identifier)+"""</td>
        <td style="text-align:center">"""+(item.category)+"""</td>
        <td style="text-align:center">"""+str(item.rank)+"""</td>
        </tr>"""
    message+="""</tbody></table></body></html>"""
    print message
    #recipients = ['kshitij.sood@saholic.com']
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
    msg = MIMEMultipart()
    msg['Subject'] = "Amazon Best Sellers" + ' - ' + str(datetime.now())
    msg['From'] = ""
    msg['To'] = ",".join(recipients)
    msg.preamble = "Amazon Best Sellers" + ' - ' + str(datetime.now())
    html_msg = MIMEText(message, 'html')
    msg.attach(html_msg)
    
    smtpServer = smtplib.SMTP('localhost')
    smtpServer.set_debuglevel(1)
    sender = 'dtr@shop2020.in'
    try:
        smtpServer.sendmail(sender, recipients, msg.as_string())
        print "Successfully sent email"
    except:
        print "Error: unable to send email."
            
            
            
def main():
    scrapeBestSellerMobiles()
    if len(bestSellers) > 0:
        resetRanks(3)
        commitBestSellers("MOBILE")
    scrapeBestSellerTablets()
    if len(bestSellers) > 0:
        resetRanks(5)
        commitBestSellers("TABLET")
    sendEmail()
        
if __name__=='__main__':
    main()