Subversion Repositories SmartDukaan

Rev

Rev 18284 | Rev 19645 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13754 kshitij.so 1 
import urllib2
2 
from BeautifulSoup import BeautifulSoup
3 
import pymongo
4 
import re
18284 kshitij.so 5 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
13828 kshitij.so 6 
from datetime import datetime
14257 kshitij.so 7 
import optparse
14379 kshitij.so 8 
import smtplib
9 
from email.mime.text import MIMEText
10 
from email.mime.multipart import MIMEMultipart
18284 kshitij.so 11 
from dtr.utils.utils import fetchResponseUsingProxy
13754 kshitij.so 12 
 
13 
con = None
14257 kshitij.so 14 
parser = optparse.OptionParser()
15 
parser.add_option("-m", "--m", dest="mongoHost",
16 
                      default="localhost",
17 
                      type="string", help="The HOST where the mongo server is running",
18 
                      metavar="mongo_host")
19 
 
20 
(options, args) = parser.parse_args()
21 
 
13754 kshitij.so 22 
bestSellers = []
13828 kshitij.so 23 
now = datetime.now()
14379 kshitij.so 24 
exceptionList = []
13754 kshitij.so 25 
 
26 
class __RankInfo:
27 
 
14379 kshitij.so 28 
    def __init__(self, identifier, rank, category):
13754 kshitij.so 29 
        self.identifier = identifier
30 
        self.rank  = rank
14379 kshitij.so 31 
        self.category = category
13754 kshitij.so 32 
 
14257 kshitij.so 33 
def get_mongo_connection(host=options.mongoHost, port=27017):
13754 kshitij.so 34 
    global con
35 
    if con is None:
36 
        print "Establishing connection %s host and port %d" %(host,port)
37 
        try:
38 
            con = pymongo.MongoClient(host, port)
39 
        except Exception, e:
18284 kshitij.so 40 
 
13754 kshitij.so 41 
            print e
42 
            return None
43 
    return con
44 
 
45 
def getSoupObject(url):
46 
    print "Getting soup object for"
47 
    print url
18284 kshitij.so 48 
    soup = None
19240 kshitij.so 49 
    response_data = fetchResponseUsingProxy(url,proxy=False)
18284 kshitij.so 50 
    print response_data
51 
    try:
52 
        page=response_data.decode("utf-8")
53 
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
54 
    except:
55 
        soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
56 
    return soup
13754 kshitij.so 57 
 
58 
def scrapeBestSellerMobiles():
59 
    global bestSellers
60 
    rank = 0
61 
    for i in [1, 21, 41, 61, 81]:
19240 kshitij.so 62 
        url = "http://www.flipkart.com/lc/pr/pv1/spotList1/spot1/productList?sid=tyy,4io&filterNone=true&start=%d&ajax=true" %(i)
63 
 
64 
 
13754 kshitij.so 65 
        soup = getSoupObject(url)
66 
        product_divs = soup.findAll('div',{'class':re.compile('.*browse-product')})
67 
        for x in product_divs:
68 
            rank  = rank +1
69 
            print rank,
70 
            print '\t',
71 
            print x['data-pid']
14379 kshitij.so 72 
            r_info = __RankInfo(x['data-pid'].strip(),rank, None)
13754 kshitij.so 73 
            bestSellers.append(r_info)
74 
 
75 
def scrapeBestSellerTablets():
76 
    global bestSellers
77 
    bestSellers = []
78 
    rank = 0
79 
    for i in [1, 21, 41, 61, 81]:
19240 kshitij.so 80 
        url = "http://www.flipkart.com/lc/pr/pv1/spotList1/spot1/productList?sid=tyy,hry&filterNone=true&start=%d&ajax=true" %(i)
13754 kshitij.so 81 
        soup = getSoupObject(url)
82 
        product_divs = soup.findAll('div',{'class':re.compile('.*browse-product')})
83 
        for x in product_divs:
84 
            rank  = rank +1
85 
            print rank,
86 
            print '\t',
87 
            print x['data-pid']
14379 kshitij.so 88 
            r_info = __RankInfo(x['data-pid'].strip(),rank, None)
13754 kshitij.so 89 
            bestSellers.append(r_info)
90 
 
14379 kshitij.so 91 
def commitBestSellers(category):
92 
    global exceptionList
13754 kshitij.so 93 
    for x in bestSellers:
94 
        print x.rank,
95 
        print '\t',
96 
        print x.identifier,
97 
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier.strip()})
98 
        print "count sku",
99 
        print '\t',
14379 kshitij.so 100 
        if len(list(col)) == 0:
101 
            x.category = category
102 
            exceptionList.append(x)
103 
        else:
104 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier.strip()}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
13754 kshitij.so 105 
 
106 
def resetRanks(category):
13828 kshitij.so 107 
    oldRankedItems = get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':2,'category':category})
108 
    for item in oldRankedItems:
109 
        get_mongo_connection().Catalog.MasterData.update({'_id':item['_id']}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
14379 kshitij.so 110 
 
111 
def sendEmail():
112 
    message="""<html>
113 
            <body>
114 
            <h3>Flipkart Best Sellers not in master</h3>
115 
            <table border="1" style="width:100%;">
116 
            <thead>
117 
            <tr><th>Identifier</th>
118 
            <th>Category</th>
119 
            <th>Rank</th>
120 
            </tr></thead>
121 
            <tbody>"""
122 
    for item in exceptionList:
123 
        message+="""<tr>
124 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
125 
        <td style="text-align:center">"""+(item.category)+"""</td>
126 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
127 
        </tr>"""
128 
    message+="""</tbody></table></body></html>"""
129 
    print message
18284 kshitij.so 130 
    recipients = ['kshitij.sood@saholic.com']
131 
    #recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
14379 kshitij.so 132 
    msg = MIMEMultipart()
133 
    msg['Subject'] = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
134 
    msg['From'] = ""
135 
    msg['To'] = ",".join(recipients)
136 
    msg.preamble = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
137 
    html_msg = MIMEText(message, 'html')
138 
    msg.attach(html_msg)
139 
 
140 
    smtpServer = smtplib.SMTP('localhost')
141 
    smtpServer.set_debuglevel(1)
142 
    sender = 'dtr@shop2020.in'
143 
    try:
144 
        smtpServer.sendmail(sender, recipients, msg.as_string())
145 
        print "Successfully sent email"
146 
    except:
147 
        print "Error: unable to send email."
13754 kshitij.so 148 
 
149 
def main():
150 
    scrapeBestSellerMobiles()
151 
    if len(bestSellers) > 0:
152 
        resetRanks('Mobiles')
14379 kshitij.so 153 
        commitBestSellers('MOBILES')
13754 kshitij.so 154 
    scrapeBestSellerTablets()
155 
    if len(bestSellers) > 0:
156 
        resetRanks('Tablets')
14379 kshitij.so 157 
        commitBestSellers('TABLETS')
158 
    sendEmail()
13754 kshitij.so 159 
 
160 
if __name__=='__main__':
161 
    main()