Subversion Repositories SmartDukaan

Rev

Rev 16464 | Rev 16512 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
16381 amit.gupta 1 
from BeautifulSoup import BeautifulSoup
2 
from datetime import datetime
3 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
4 
from email.mime.multipart import MIMEMultipart
5 
from email.mime.text import MIMEText
6 
from scripts.AmazonBestSellers import getSoupObject
7 
import json
8 
import optparse
9 
import pymongo
10 
import re
11 
import smtplib
12 
import urllib2
13 
 
14 
con = None
15 
parser = optparse.OptionParser()
16 
parser.add_option("-m", "--m", dest="mongoHost",
17 
                      default="localhost",
18 
                      type="string", help="The HOST where the mongo server is running",
19 
                      metavar="mongo_host")
20 
 
21 
(options, args) = parser.parse_args()
22 
 
23 
 
24 
exceptionList = []
25 
bestSellers = []
26 
now = datetime.now()
27 
 
28 
 
29 
class __RankInfo:
30 
 
31 
    def __init__(self, identifier, rank, category):
32 
        self.identifier = identifier
33 
        self.rank  = rank
34 
        self.category = category
35 
 
36 
def get_mongo_connection(host=options.mongoHost, port=27017):
37 
    global con
38 
    if con is None:
39 
        print "Establishing connection %s host and port %d" %(host,port)
40 
        try:
41 
            con = pymongo.MongoClient(host, port)
42 
        except Exception, e:
43 
            print e
44 
            return None
45 
    return con
46 
 
47 
 
48 
 
49 
def scrapeBestSellerMobiles():
50 
    global bestSellers
51 
    rank = 0
52 
    mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'
53 
    for i in range(1,5):
54 
        data = fetchResponseUsingProxy(mobileCategoryUrl%(i))
55 
        jsonResponse = json.loads(data)
56 
        for jsonProduct in jsonResponse['grid_layout']:
57 
            rank = rank + 1
58 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
59 
            r_info = __RankInfo(identifier,rank, None)
60 
            print rank, identifier
61 
            bestSellers.append(r_info)
62 
 
63 
 
64 
def commitBestSellers(category):
65 
    global exceptionList
66 
    print "Rank",
67 
    print '\t',
68 
    print 'Identifier'
69 
    for x in bestSellers:
70 
        print x.rank,
71 
        print '\t',
72 
        print x.identifier,
16472 kshitij.so 73 
        print '\t',
74 
        col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6}))
75 
        print "count sku",len(col)
76 
        print '\n'
77 
        if len(col) == 0:
16381 amit.gupta 78 
            x.category = category
79 
            exceptionList.append(x)
80 
        else:
81 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})
82 
 
83 
def scrapeBestSellerTablets():
84 
    global bestSellers
16464 kshitij.so 85 
    bestSellers = []
16381 amit.gupta 86 
    rank = 0
87 
    bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'
88 
    for i in range(1,5):
89 
        data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))
90 
        jsonResponse = json.loads(data)
91 
        for jsonProduct in jsonResponse['grid_layout']:
92 
            rank = rank + 1
93 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
94 
            r_info = __RankInfo(identifier,rank, None)
95 
            print rank, identifier
96 
            bestSellers.append(r_info)
97 
 
98 
def resetRanks(category_id):
99 
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)
100 
 
101 
def sendEmail():
102 
    message="""<html>
103 
            <body>
104 
            <h3>PayTM Best Sellers not in master</h3>
105 
            <table border="1" style="width:100%;">
106 
            <thead>
107 
            <tr><th>Identifier</th>
108 
            <th>Category</th>
109 
            <th>Rank</th>
110 
            </tr></thead>
111 
            <tbody>"""
112 
    for item in exceptionList:
113 
        message+="""<tr>
114 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
115 
        <td style="text-align:center">"""+(item.category)+"""</td>
116 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
117 
        </tr>"""
118 
    message+="""</tbody></table></body></html>"""
119 
    print message
120 
    #recipients = ['amit.gupta@saholic.com']
121 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com','amit.gupta@saholic.com']
122 
    msg = MIMEMultipart()
16464 kshitij.so 123 
    msg['Subject'] = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 124 
    msg['From'] = ""
125 
    msg['To'] = ",".join(recipients)
16464 kshitij.so 126 
    msg.preamble = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 127 
    html_msg = MIMEText(message, 'html')
128 
    msg.attach(html_msg)
129 
 
130 
    smtpServer = smtplib.SMTP('localhost')
131 
    smtpServer.set_debuglevel(1)
132 
    sender = 'dtr@shop2020.in'
133 
    try:
134 
        smtpServer.sendmail(sender, recipients, msg.as_string())
135 
        print "Successfully sent email"
136 
    except:
137 
        print "Error: unable to send email."
138 
 
139 
 
140 
 
141 
def main():
142 
    scrapeBestSellerMobiles()
143 
    if len(bestSellers) > 0:
16464 kshitij.so 144 
        resetRanks(3)
16381 amit.gupta 145 
        commitBestSellers("MOBILE")
146 
    scrapeBestSellerTablets()
147 
    if len(bestSellers) > 0:
148 
        resetRanks(5)
149 
        commitBestSellers("TABLET")
150 
    sendEmail()
151 
 
152 
if __name__=='__main__':
153 
    main()