Subversion Repositories SmartDukaan

Rev

Rev 17292 | Rev 18251 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
16381 amit.gupta 1 
from BeautifulSoup import BeautifulSoup
2 
from datetime import datetime
3 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
4 
from email.mime.multipart import MIMEMultipart
5 
from email.mime.text import MIMEText
6 
from scripts.AmazonBestSellers import getSoupObject
7 
import json
8 
import optparse
9 
import pymongo
10 
import re
11 
import smtplib
12 
import urllib2
16512 kshitij.so 13 
from dtr.utils.PaytmOfferScraper import fetchOffers
16381 amit.gupta 14 
 
15 
con = None
16 
parser = optparse.OptionParser()
17 
parser.add_option("-m", "--m", dest="mongoHost",
18 
                      default="localhost",
19 
                      type="string", help="The HOST where the mongo server is running",
20 
                      metavar="mongo_host")
21 
 
22 
(options, args) = parser.parse_args()
23 
 
24 
 
25 
exceptionList = []
26 
bestSellers = []
27 
now = datetime.now()
28 
 
29 
 
30 
class __RankInfo:
31 
 
16512 kshitij.so 32 
    def __init__(self, identifier, rank, category, available_price, gross_price, in_stock, coupon, thumbnail, source_product_name, marketPlaceUrl, cod):
16381 amit.gupta 33 
        self.identifier = identifier
34 
        self.rank  = rank
35 
        self.category = category
16512 kshitij.so 36 
        self.available_price = available_price
37 
        self.gross_price = gross_price
38 
        self.in_stock = in_stock
39 
        self.coupon = coupon
40 
        self.thumbnail = thumbnail
41 
        self.source_product_name = source_product_name
42 
        self.marketPlaceUrl = marketPlaceUrl
43 
        self.cod = cod
44 
 
45 
 
46 
 
16381 amit.gupta 47 
 
48 
def get_mongo_connection(host=options.mongoHost, port=27017):
49 
    global con
50 
    if con is None:
51 
        print "Establishing connection %s host and port %d" %(host,port)
52 
        try:
53 
            con = pymongo.MongoClient(host, port)
54 
        except Exception, e:
55 
            print e
56 
            return None
57 
    return con
58 
 
59 
 
60 
 
61 
def scrapeBestSellerMobiles():
62 
    global bestSellers
63 
    rank = 0
64 
    mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'
65 
    for i in range(1,5):
66 
        data = fetchResponseUsingProxy(mobileCategoryUrl%(i))
67 
        jsonResponse = json.loads(data)
68 
        for jsonProduct in jsonResponse['grid_layout']:
69 
            rank = rank + 1
17293 kshitij.so 70 
            identifier = str(jsonProduct['complex_product_id'])
16513 kshitij.so 71 
            r_info = __RankInfo(identifier,rank, None, None,None,None,None,None,None,None,None)
16381 amit.gupta 72 
            print rank, identifier
73 
            bestSellers.append(r_info)
74 
 
75 
 
76 
def commitBestSellers(category):
77 
    global exceptionList
78 
    print "Rank",
79 
    print '\t',
80 
    print 'Identifier'
81 
    for x in bestSellers:
82 
        print x.rank,
83 
        print '\t',
84 
        print x.identifier,
16472 kshitij.so 85 
        print '\t',
86 
        col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6}))
87 
        print "count sku",len(col)
88 
        print '\n'
89 
        if len(col) == 0:
16381 amit.gupta 90 
            x.category = category
91 
            exceptionList.append(x)
92 
        else:
93 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})
94 
 
95 
def scrapeBestSellerTablets():
96 
    global bestSellers
16464 kshitij.so 97 
    bestSellers = []
16381 amit.gupta 98 
    rank = 0
99 
    bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'
100 
    for i in range(1,5):
101 
        data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))
102 
        jsonResponse = json.loads(data)
103 
        for jsonProduct in jsonResponse['grid_layout']:
104 
            rank = rank + 1
105 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
16513 kshitij.so 106 
            r_info = __RankInfo(identifier,rank,  None, None,None,None,None,None,None,None,None)
16381 amit.gupta 107 
            print rank, identifier
108 
            bestSellers.append(r_info)
109 
 
110 
def resetRanks(category_id):
111 
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)
112 
 
113 
def sendEmail():
114 
    message="""<html>
115 
            <body>
116 
            <h3>PayTM Best Sellers not in master</h3>
117 
            <table border="1" style="width:100%;">
118 
            <thead>
119 
            <tr><th>Identifier</th>
120 
            <th>Category</th>
121 
            <th>Rank</th>
16512 kshitij.so 122 
            <th>Available_price</th>
123 
            <th>Gross_price</th>
124 
            <th>In_stock</th>
125 
            <th>Coupon</th>
126 
            <th>Thumbnail</th>
127 
            <th>Source_product_name</th>
128 
            <th>MarketPlaceUrl</th>
129 
            <th>Cod</th>
16381 amit.gupta 130 
            </tr></thead>
131 
            <tbody>"""
132 
    for item in exceptionList:
16518 kshitij.so 133 
        try:
134 
            message+="""<tr>
135 
            <td style="text-align:center">"""+(item.identifier)+"""</td>
136 
            <td style="text-align:center">"""+(item.category)+"""</td>
137 
            <td style="text-align:center">"""+str(item.rank)+"""</td>
138 
            <td style="text-align:center">"""+str(item.available_price)+"""</td>
139 
            <td style="text-align:center">"""+str(item.gross_price)+"""</td>
140 
            <td style="text-align:center">"""+str(item.in_stock)+"""</td>
141 
            <td style="text-align:center">"""+str(item.coupon)+"""</td>
142 
            <td style="text-align:center">"""+str(item.thumbnail)+"""</td>
143 
            <td style="text-align:center">"""+str(item.source_product_name)+"""</td>
144 
            <td style="text-align:center">"""+str(item.marketPlaceUrl)+"""</td>
145 
            <td style="text-align:center">"""+str(item.cod)+"""</td>
146 
            </tr>"""
147 
        except:
148 
            continue
16381 amit.gupta 149 
    message+="""</tbody></table></body></html>"""
150 
    print message
151 
    #recipients = ['amit.gupta@saholic.com']
152 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com','amit.gupta@saholic.com']
153 
    msg = MIMEMultipart()
16464 kshitij.so 154 
    msg['Subject'] = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 155 
    msg['From'] = ""
156 
    msg['To'] = ",".join(recipients)
16464 kshitij.so 157 
    msg.preamble = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 158 
    html_msg = MIMEText(message, 'html')
159 
    msg.attach(html_msg)
160 
 
161 
    smtpServer = smtplib.SMTP('localhost')
162 
    smtpServer.set_debuglevel(1)
163 
    sender = 'dtr@shop2020.in'
164 
    try:
165 
        smtpServer.sendmail(sender, recipients, msg.as_string())
166 
        print "Successfully sent email"
167 
    except:
168 
        print "Error: unable to send email."
16512 kshitij.so 169 
 
170 
def addExtraAttributes():
171 
    for e in exceptionList:
172 
        url = "https://catalog.paytm.com/v1/mobile/product/%s" %(e.identifier.strip())
173 
        try:
174 
            response_data = fetchResponseUsingProxy(url,  proxy=False)
175 
        except:
176 
            continue
177 
        input_json = json.loads(response_data)
178 
        offerPrice = float(input_json['offer_price'])
179 
        e.cod = int(input_json['pay_type_supported'].get('COD'))
180 
        offerUrl = (input_json['offer_url'])
181 
        e.in_stock = (input_json['instock'])
182 
        try:
183 
            offers = fetchOffers(offerUrl)
184 
        except:
185 
            continue
186 
        bestOffer = {}
16514 kshitij.so 187 
        e.gross_price = float(offerPrice)
16512 kshitij.so 188 
        effective_price = offerPrice
189 
        coupon = ""
190 
        for offer_data in offers.get('codes'):
191 
            if effective_price > offer_data.get('effective_price'):
192 
                effective_price = offer_data.get('effective_price')
193 
                bestOffer = offer_data
194 
                coupon = bestOffer.get('code')
195 
        e.source_product_name = input_json['name']
196 
        e.thumbnail = input_json['thumbnail']
197 
        e.marketPlaceUrl = input_json['shareurl']
198 
        e.coupon = coupon
16514 kshitij.so 199 
        e.available_price = float(effective_price)
16512 kshitij.so 200 
 
16381 amit.gupta 201 
 
202 
 
203 
 
204 
def main():
205 
    scrapeBestSellerMobiles()
206 
    if len(bestSellers) > 0:
16464 kshitij.so 207 
        resetRanks(3)
16381 amit.gupta 208 
        commitBestSellers("MOBILE")
209 
    scrapeBestSellerTablets()
210 
    if len(bestSellers) > 0:
211 
        resetRanks(5)
212 
        commitBestSellers("TABLET")
16512 kshitij.so 213 
    addExtraAttributes()
16381 amit.gupta 214 
    sendEmail()
215 
 
216 
if __name__=='__main__':
217 
    main()