Subversion Repositories SmartDukaan

Rev

Rev 16381 | Rev 16472 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
16381 amit.gupta 1 
from BeautifulSoup import BeautifulSoup
2 
from datetime import datetime
3 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
4 
from email.mime.multipart import MIMEMultipart
5 
from email.mime.text import MIMEText
6 
from scripts.AmazonBestSellers import getSoupObject
7 
import json
8 
import optparse
9 
import pymongo
10 
import re
11 
import smtplib
12 
import urllib2
13 
 
14 
con = None
15 
parser = optparse.OptionParser()
16 
parser.add_option("-m", "--m", dest="mongoHost",
17 
                      default="localhost",
18 
                      type="string", help="The HOST where the mongo server is running",
19 
                      metavar="mongo_host")
20 
 
21 
(options, args) = parser.parse_args()
22 
 
23 
 
24 
exceptionList = []
25 
bestSellers = []
26 
now = datetime.now()
27 
 
28 
 
29 
class __RankInfo:
30 
 
31 
    def __init__(self, identifier, rank, category):
32 
        self.identifier = identifier
33 
        self.rank  = rank
34 
        self.category = category
35 
 
36 
def get_mongo_connection(host=options.mongoHost, port=27017):
37 
    global con
38 
    if con is None:
39 
        print "Establishing connection %s host and port %d" %(host,port)
40 
        try:
41 
            con = pymongo.MongoClient(host, port)
42 
        except Exception, e:
43 
            print e
44 
            return None
45 
    return con
46 
 
47 
 
48 
 
49 
def scrapeBestSellerMobiles():
50 
    global bestSellers
51 
    rank = 0
52 
    mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'
53 
    for i in range(1,5):
54 
        data = fetchResponseUsingProxy(mobileCategoryUrl%(i))
55 
        jsonResponse = json.loads(data)
56 
        for jsonProduct in jsonResponse['grid_layout']:
57 
            rank = rank + 1
58 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
59 
            r_info = __RankInfo(identifier,rank, None)
60 
            print rank, identifier
61 
            bestSellers.append(r_info)
62 
 
63 
 
64 
def commitBestSellers(category):
65 
    global exceptionList
66 
    print "Rank",
67 
    print '\t',
68 
    print 'Identifier'
69 
    for x in bestSellers:
70 
        print x.rank,
71 
        print '\t',
72 
        print x.identifier,
73 
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6})
16464 kshitij.so 74 
        print "count sku",len(list(col))
16381 amit.gupta 75 
        print '\n',
76 
        if len(list(col)) == 0:
77 
            x.category = category
78 
            exceptionList.append(x)
79 
        else:
80 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})
81 
 
82 
def scrapeBestSellerTablets():
83 
    global bestSellers
16464 kshitij.so 84 
    bestSellers = []
16381 amit.gupta 85 
    rank = 0
86 
    bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'
87 
    for i in range(1,5):
88 
        data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))
89 
        jsonResponse = json.loads(data)
90 
        for jsonProduct in jsonResponse['grid_layout']:
91 
            rank = rank + 1
92 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
93 
            r_info = __RankInfo(identifier,rank, None)
94 
            print rank, identifier
95 
            bestSellers.append(r_info)
96 
 
97 
def resetRanks(category_id):
98 
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)
99 
 
100 
def sendEmail():
101 
    message="""<html>
102 
            <body>
103 
            <h3>PayTM Best Sellers not in master</h3>
104 
            <table border="1" style="width:100%;">
105 
            <thead>
106 
            <tr><th>Identifier</th>
107 
            <th>Category</th>
108 
            <th>Rank</th>
109 
            </tr></thead>
110 
            <tbody>"""
111 
    for item in exceptionList:
112 
        message+="""<tr>
113 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
114 
        <td style="text-align:center">"""+(item.category)+"""</td>
115 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
116 
        </tr>"""
117 
    message+="""</tbody></table></body></html>"""
118 
    print message
119 
    #recipients = ['amit.gupta@saholic.com']
120 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com','amit.gupta@saholic.com']
121 
    msg = MIMEMultipart()
16464 kshitij.so 122 
    msg['Subject'] = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 123 
    msg['From'] = ""
124 
    msg['To'] = ",".join(recipients)
16464 kshitij.so 125 
    msg.preamble = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 126 
    html_msg = MIMEText(message, 'html')
127 
    msg.attach(html_msg)
128 
 
129 
    smtpServer = smtplib.SMTP('localhost')
130 
    smtpServer.set_debuglevel(1)
131 
    sender = 'dtr@shop2020.in'
132 
    try:
133 
        smtpServer.sendmail(sender, recipients, msg.as_string())
134 
        print "Successfully sent email"
135 
    except:
136 
        print "Error: unable to send email."
137 
 
138 
 
139 
 
140 
def main():
141 
    scrapeBestSellerMobiles()
142 
    if len(bestSellers) > 0:
16464 kshitij.so 143 
        resetRanks(3)
16381 amit.gupta 144 
        commitBestSellers("MOBILE")
145 
    scrapeBestSellerTablets()
146 
    if len(bestSellers) > 0:
147 
        resetRanks(5)
148 
        commitBestSellers("TABLET")
149 
    sendEmail()
150 
 
151 
if __name__=='__main__':
152 
    main()