Subversion Repositories SmartDukaan

Rev

Rev 16464 | Go to most recent revision | Details | Last modification | View Log | RSS feed

Rev Author Line No. Line
16381 amit.gupta 1 
from BeautifulSoup import BeautifulSoup
2 
from datetime import datetime
3 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
4 
from email.mime.multipart import MIMEMultipart
5 
from email.mime.text import MIMEText
6 
from scripts.AmazonBestSellers import getSoupObject
7 
import json
8 
import optparse
9 
import pymongo
10 
import re
11 
import smtplib
12 
import urllib2
13 
 
14 
con = None
15 
parser = optparse.OptionParser()
16 
parser.add_option("-m", "--m", dest="mongoHost",
17 
                      default="localhost",
18 
                      type="string", help="The HOST where the mongo server is running",
19 
                      metavar="mongo_host")
20 
 
21 
(options, args) = parser.parse_args()
22 
 
23 
 
24 
exceptionList = []
25 
asin_regex = r'/([A-Z0-9]{10})'
26 
bestSellers = []
27 
now = datetime.now()
28 
 
29 
 
30 
class __RankInfo:
31 
 
32 
    def __init__(self, identifier, rank, category):
33 
        self.identifier = identifier
34 
        self.rank  = rank
35 
        self.category = category
36 
 
37 
def get_mongo_connection(host=options.mongoHost, port=27017):
38 
    global con
39 
    if con is None:
40 
        print "Establishing connection %s host and port %d" %(host,port)
41 
        try:
42 
            con = pymongo.MongoClient(host, port)
43 
        except Exception, e:
44 
            print e
45 
            return None
46 
    return con
47 
 
48 
 
49 
 
50 
def scrapeBestSellerMobiles():
51 
    global bestSellers
52 
    rank = 0
53 
    mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'
54 
    for i in range(1,5):
55 
        data = fetchResponseUsingProxy(mobileCategoryUrl%(i))
56 
        jsonResponse = json.loads(data)
57 
        for jsonProduct in jsonResponse['grid_layout']:
58 
            rank = rank + 1
59 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
60 
            r_info = __RankInfo(identifier,rank, None)
61 
            print rank, identifier
62 
            bestSellers.append(r_info)
63 
 
64 
 
65 
def commitBestSellers(category):
66 
    global exceptionList
67 
    print "Rank",
68 
    print '\t',
69 
    print 'Identifier'
70 
    for x in bestSellers:
71 
        print x.rank,
72 
        print '\t',
73 
        print x.identifier,
74 
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6})
75 
        print "count sku",
76 
        print '\n',
77 
        if len(list(col)) == 0:
78 
            x.category = category
79 
            exceptionList.append(x)
80 
        else:
81 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})
82 
 
83 
def scrapeBestSellerTablets():
84 
    global bestSellers
85 
    rank = 0
86 
    bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'
87 
    for i in range(1,5):
88 
        data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))
89 
        jsonResponse = json.loads(data)
90 
        for jsonProduct in jsonResponse['grid_layout']:
91 
            rank = rank + 1
92 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
93 
            r_info = __RankInfo(identifier,rank, None)
94 
            print rank, identifier
95 
            bestSellers.append(r_info)
96 
 
97 
def resetRanks(category_id):
98 
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)
99 
 
100 
def sendEmail():
101 
    message="""<html>
102 
            <body>
103 
            <h3>PayTM Best Sellers not in master</h3>
104 
            <table border="1" style="width:100%;">
105 
            <thead>
106 
            <tr><th>Identifier</th>
107 
            <th>Category</th>
108 
            <th>Rank</th>
109 
            </tr></thead>
110 
            <tbody>"""
111 
    for item in exceptionList:
112 
        message+="""<tr>
113 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
114 
        <td style="text-align:center">"""+(item.category)+"""</td>
115 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
116 
        </tr>"""
117 
    message+="""</tbody></table></body></html>"""
118 
    print message
119 
    #recipients = ['amit.gupta@saholic.com']
120 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com','amit.gupta@saholic.com']
121 
    msg = MIMEMultipart()
122 
    msg['Subject'] = "Amazon Best Sellers" + ' - ' + str(datetime.now())
123 
    msg['From'] = ""
124 
    msg['To'] = ",".join(recipients)
125 
    msg.preamble = "Amazon Best Sellers" + ' - ' + str(datetime.now())
126 
    html_msg = MIMEText(message, 'html')
127 
    msg.attach(html_msg)
128 
 
129 
    smtpServer = smtplib.SMTP('localhost')
130 
    smtpServer.set_debuglevel(1)
131 
    sender = 'dtr@shop2020.in'
132 
    try:
133 
        smtpServer.sendmail(sender, recipients, msg.as_string())
134 
        print "Successfully sent email"
135 
    except:
136 
        print "Error: unable to send email."
137 
 
138 
 
139 
 
140 
def main():
141 
    scrapeBestSellerMobiles()
142 
    if len(bestSellers) > 0:
143 
        resetRanks(6)
144 
        commitBestSellers("MOBILE")
145 
    scrapeBestSellerTablets()
146 
    if len(bestSellers) > 0:
147 
        resetRanks(5)
148 
        commitBestSellers("TABLET")
149 
    sendEmail()
150 
 
151 
if __name__=='__main__':
152 
    main()