Subversion Repositories SmartDukaan

Rev

Rev 20172 | Rev 21135 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
16381 amit.gupta 1 
from BeautifulSoup import BeautifulSoup
2 
from datetime import datetime
3 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
4 
from email.mime.multipart import MIMEMultipart
5 
from email.mime.text import MIMEText
6 
import json
7 
import optparse
8 
import pymongo
9 
import re
10 
import smtplib
11 
import urllib2
16512 kshitij.so 12 
from dtr.utils.PaytmOfferScraper import fetchOffers
16381 amit.gupta 13 
 
14 
con = None
15 
parser = optparse.OptionParser()
16 
parser.add_option("-m", "--m", dest="mongoHost",
17 
                      default="localhost",
18 
                      type="string", help="The HOST where the mongo server is running",
19 
                      metavar="mongo_host")
20 
 
21 
(options, args) = parser.parse_args()
22 
 
23 
 
24 
exceptionList = []
25 
bestSellers = []
26 
now = datetime.now()
27 
 
28 
 
29 
class __RankInfo:
30 
 
16512 kshitij.so 31 
    def __init__(self, identifier, rank, category, available_price, gross_price, in_stock, coupon, thumbnail, source_product_name, marketPlaceUrl, cod):
16381 amit.gupta 32 
        self.identifier = identifier
33 
        self.rank  = rank
34 
        self.category = category
16512 kshitij.so 35 
        self.available_price = available_price
36 
        self.gross_price = gross_price
37 
        self.in_stock = in_stock
38 
        self.coupon = coupon
39 
        self.thumbnail = thumbnail
40 
        self.source_product_name = source_product_name
41 
        self.marketPlaceUrl = marketPlaceUrl
42 
        self.cod = cod
43 
 
44 
 
45 
 
16381 amit.gupta 46 
 
47 
def get_mongo_connection(host=options.mongoHost, port=27017):
48 
    global con
49 
    if con is None:
50 
        print "Establishing connection %s host and port %d" %(host,port)
51 
        try:
52 
            con = pymongo.MongoClient(host, port)
53 
        except Exception, e:
54 
            print e
55 
            return None
56 
    return con
57 
 
58 
 
59 
 
60 
def scrapeBestSellerMobiles():
61 
    global bestSellers
62 
    rank = 0
63 
    mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'
64 
    for i in range(1,5):
65 
        data = fetchResponseUsingProxy(mobileCategoryUrl%(i))
66 
        jsonResponse = json.loads(data)
67 
        for jsonProduct in jsonResponse['grid_layout']:
68 
            rank = rank + 1
17293 kshitij.so 69 
            identifier = str(jsonProduct['complex_product_id'])
16513 kshitij.so 70 
            r_info = __RankInfo(identifier,rank, None, None,None,None,None,None,None,None,None)
16381 amit.gupta 71 
            print rank, identifier
72 
            bestSellers.append(r_info)
73 
 
74 
 
75 
def commitBestSellers(category):
76 
    global exceptionList
77 
    print "Rank",
78 
    print '\t',
79 
    print 'Identifier'
80 
    for x in bestSellers:
81 
        print x.rank,
82 
        print '\t',
83 
        print x.identifier,
16472 kshitij.so 84 
        print '\t',
85 
        col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6}))
86 
        print "count sku",len(col)
87 
        print '\n'
88 
        if len(col) == 0:
16381 amit.gupta 89 
            x.category = category
90 
            exceptionList.append(x)
91 
        else:
92 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})
93 
 
94 
def scrapeBestSellerTablets():
95 
    global bestSellers
16464 kshitij.so 96 
    bestSellers = []
16381 amit.gupta 97 
    rank = 0
98 
    bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'
99 
    for i in range(1,5):
100 
        data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))
101 
        jsonResponse = json.loads(data)
102 
        for jsonProduct in jsonResponse['grid_layout']:
103 
            rank = rank + 1
18251 kshitij.so 104 
            identifier = str(jsonProduct['complex_product_id'])
16513 kshitij.so 105 
            r_info = __RankInfo(identifier,rank,  None, None,None,None,None,None,None,None,None)
16381 amit.gupta 106 
            print rank, identifier
107 
            bestSellers.append(r_info)
108 
 
109 
def resetRanks(category_id):
110 
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)
111 
 
112 
def sendEmail():
113 
    message="""<html>
114 
            <body>
115 
            <h3>PayTM Best Sellers not in master</h3>
116 
            <table border="1" style="width:100%;">
117 
            <thead>
118 
            <tr><th>Identifier</th>
119 
            <th>Category</th>
120 
            <th>Rank</th>
16512 kshitij.so 121 
            <th>Available_price</th>
122 
            <th>Gross_price</th>
123 
            <th>In_stock</th>
124 
            <th>Coupon</th>
125 
            <th>Thumbnail</th>
126 
            <th>Source_product_name</th>
127 
            <th>MarketPlaceUrl</th>
128 
            <th>Cod</th>
16381 amit.gupta 129 
            </tr></thead>
130 
            <tbody>"""
131 
    for item in exceptionList:
16518 kshitij.so 132 
        try:
133 
            message+="""<tr>
134 
            <td style="text-align:center">"""+(item.identifier)+"""</td>
135 
            <td style="text-align:center">"""+(item.category)+"""</td>
136 
            <td style="text-align:center">"""+str(item.rank)+"""</td>
137 
            <td style="text-align:center">"""+str(item.available_price)+"""</td>
138 
            <td style="text-align:center">"""+str(item.gross_price)+"""</td>
139 
            <td style="text-align:center">"""+str(item.in_stock)+"""</td>
140 
            <td style="text-align:center">"""+str(item.coupon)+"""</td>
141 
            <td style="text-align:center">"""+str(item.thumbnail)+"""</td>
142 
            <td style="text-align:center">"""+str(item.source_product_name)+"""</td>
143 
            <td style="text-align:center">"""+str(item.marketPlaceUrl)+"""</td>
144 
            <td style="text-align:center">"""+str(item.cod)+"""</td>
145 
            </tr>"""
146 
        except:
147 
            continue
16381 amit.gupta 148 
    message+="""</tbody></table></body></html>"""
149 
    print message
150 
    #recipients = ['amit.gupta@saholic.com']
20172 aman.kumar 151 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com', 'khushal.bhatia@saholic.com', 'ritesh.chauhan@saholic.com','amit.gupta@saholic.com']
16381 amit.gupta 152 
    msg = MIMEMultipart()
16464 kshitij.so 153 
    msg['Subject'] = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 154 
    msg['From'] = ""
155 
    msg['To'] = ",".join(recipients)
16464 kshitij.so 156 
    msg.preamble = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 157 
    html_msg = MIMEText(message, 'html')
158 
    msg.attach(html_msg)
159 
 
160 
    smtpServer = smtplib.SMTP('localhost')
161 
    smtpServer.set_debuglevel(1)
162 
    sender = 'dtr@shop2020.in'
163 
    try:
164 
        smtpServer.sendmail(sender, recipients, msg.as_string())
165 
        print "Successfully sent email"
166 
    except:
167 
        print "Error: unable to send email."
16512 kshitij.so 168 
 
169 
def addExtraAttributes():
170 
    for e in exceptionList:
171 
        url = "https://catalog.paytm.com/v1/mobile/product/%s" %(e.identifier.strip())
172 
        try:
173 
            response_data = fetchResponseUsingProxy(url,  proxy=False)
174 
        except:
175 
            continue
176 
        input_json = json.loads(response_data)
177 
        offerPrice = float(input_json['offer_price'])
178 
        e.cod = int(input_json['pay_type_supported'].get('COD'))
179 
        offerUrl = (input_json['offer_url'])
180 
        e.in_stock = (input_json['instock'])
181 
        try:
182 
            offers = fetchOffers(offerUrl)
183 
        except:
184 
            continue
185 
        bestOffer = {}
16514 kshitij.so 186 
        e.gross_price = float(offerPrice)
16512 kshitij.so 187 
        effective_price = offerPrice
188 
        coupon = ""
189 
        for offer_data in offers.get('codes'):
190 
            if effective_price > offer_data.get('effective_price'):
191 
                effective_price = offer_data.get('effective_price')
192 
                bestOffer = offer_data
193 
                coupon = bestOffer.get('code')
194 
        e.source_product_name = input_json['name']
195 
        e.thumbnail = input_json['thumbnail']
196 
        e.marketPlaceUrl = input_json['shareurl']
197 
        e.coupon = coupon
16514 kshitij.so 198 
        e.available_price = float(effective_price)
16512 kshitij.so 199 
 
16381 amit.gupta 200 
 
201 
 
202 
 
203 
def main():
204 
    scrapeBestSellerMobiles()
205 
    if len(bestSellers) > 0:
16464 kshitij.so 206 
        resetRanks(3)
16381 amit.gupta 207 
        commitBestSellers("MOBILE")
208 
    scrapeBestSellerTablets()
209 
    if len(bestSellers) > 0:
210 
        resetRanks(5)
211 
        commitBestSellers("TABLET")
16512 kshitij.so 212 
    addExtraAttributes()
16381 amit.gupta 213 
    sendEmail()
214 
 
215 
if __name__=='__main__':
216 
    main()