Subversion Repositories SmartDukaan

Rev

Rev 16517 | Rev 16519 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
16381 amit.gupta 1 
from BeautifulSoup import BeautifulSoup
2 
from datetime import datetime
3 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
4 
from email.mime.multipart import MIMEMultipart
5 
from email.mime.text import MIMEText
6 
from scripts.AmazonBestSellers import getSoupObject
7 
import json
8 
import optparse
9 
import pymongo
10 
import re
11 
import smtplib
12 
import urllib2
16512 kshitij.so 13 
from dtr.utils.PaytmOfferScraper import fetchOffers
16381 amit.gupta 14 
 
15 
con = None
16 
parser = optparse.OptionParser()
17 
parser.add_option("-m", "--m", dest="mongoHost",
18 
                      default="localhost",
19 
                      type="string", help="The HOST where the mongo server is running",
20 
                      metavar="mongo_host")
21 
 
22 
(options, args) = parser.parse_args()
23 
 
24 
 
25 
exceptionList = []
26 
bestSellers = []
27 
now = datetime.now()
28 
 
29 
 
30 
class __RankInfo:
31 
 
16512 kshitij.so 32 
    def __init__(self, identifier, rank, category, available_price, gross_price, in_stock, coupon, thumbnail, source_product_name, marketPlaceUrl, cod):
16381 amit.gupta 33 
        self.identifier = identifier
34 
        self.rank  = rank
35 
        self.category = category
16512 kshitij.so 36 
        self.available_price = available_price
37 
        self.gross_price = gross_price
38 
        self.in_stock = in_stock
39 
        self.coupon = coupon
40 
        self.thumbnail = thumbnail
41 
        self.source_product_name = source_product_name
42 
        self.marketPlaceUrl = marketPlaceUrl
43 
        self.cod = cod
44 
 
45 
 
46 
 
16381 amit.gupta 47 
 
48 
def get_mongo_connection(host=options.mongoHost, port=27017):
49 
    global con
50 
    if con is None:
51 
        print "Establishing connection %s host and port %d" %(host,port)
52 
        try:
53 
            con = pymongo.MongoClient(host, port)
54 
        except Exception, e:
55 
            print e
56 
            return None
57 
    return con
58 
 
59 
 
60 
 
61 
def scrapeBestSellerMobiles():
62 
    global bestSellers
63 
    rank = 0
64 
    mobileCategoryUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/mobiles?page_count=%d&items_per_page=25&sort_popular=1&cat_tree=1'
65 
    for i in range(1,5):
66 
        data = fetchResponseUsingProxy(mobileCategoryUrl%(i))
67 
        jsonResponse = json.loads(data)
68 
        for jsonProduct in jsonResponse['grid_layout']:
69 
            rank = rank + 1
70 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
16513 kshitij.so 71 
            r_info = __RankInfo(identifier,rank, None, None,None,None,None,None,None,None,None)
16381 amit.gupta 72 
            print rank, identifier
73 
            bestSellers.append(r_info)
74 
 
75 
 
76 
def commitBestSellers(category):
77 
    global exceptionList
78 
    print "Rank",
79 
    print '\t',
80 
    print 'Identifier'
81 
    for x in bestSellers:
82 
        print x.rank,
83 
        print '\t',
84 
        print x.identifier,
16472 kshitij.so 85 
        print '\t',
86 
        col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier, 'source_id':6}))
87 
        print "count sku",len(col)
88 
        print '\n'
89 
        if len(col) == 0:
16381 amit.gupta 90 
            x.category = category
91 
            exceptionList.append(x)
92 
        else:
93 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier, 'source_id':6 }, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}})
94 
 
95 
def scrapeBestSellerTablets():
96 
    global bestSellers
16464 kshitij.so 97 
    bestSellers = []
16381 amit.gupta 98 
    rank = 0
99 
    bestSellerTabletsUrl = 'https://catalog.paytm.com/v1/g/electronics/mobile-accessories/headsets?page_count=%d&items_per_page=25&sort_popular=1'
100 
    for i in range(1,5):
101 
        data = fetchResponseUsingProxy(bestSellerTabletsUrl%(i))
102 
        jsonResponse = json.loads(data)
103 
        for jsonProduct in jsonResponse['grid_layout']:
104 
            rank = rank + 1
105 
            identifier = jsonProduct["url"].split("?")[0].split("/")[-1]
16513 kshitij.so 106 
            r_info = __RankInfo(identifier,rank,  None, None,None,None,None,None,None,None,None)
16381 amit.gupta 107 
            print rank, identifier
108 
            bestSellers.append(r_info)
109 
 
110 
def resetRanks(category_id):
111 
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':6,'category_id':category_id}, {'$set':{'rank':0}}, upsert=False, multi=True)
112 
 
113 
def sendEmail():
114 
    message="""<html>
115 
            <body>
116 
            <h3>PayTM Best Sellers not in master</h3>
117 
            <table border="1" style="width:100%;">
118 
            <thead>
119 
            <tr><th>Identifier</th>
120 
            <th>Category</th>
121 
            <th>Rank</th>
16512 kshitij.so 122 
            <th>Available_price</th>
123 
            <th>Gross_price</th>
124 
            <th>In_stock</th>
125 
            <th>Coupon</th>
126 
            <th>Thumbnail</th>
127 
            <th>Source_product_name</th>
128 
            <th>MarketPlaceUrl</th>
129 
            <th>Cod</th>
16381 amit.gupta 130 
            </tr></thead>
131 
            <tbody>"""
132 
    for item in exceptionList:
16516 kshitij.so 133 
        print item.identifier
134 
        print item.category
135 
        print item.rank
136 
        print item.available_price
137 
        print item.gross_price
138 
        print item.in_stock
139 
        print item.coupon
140 
        print item.thumbnail
141 
        print item.source_product_name
142 
        print item.marketPlaceUrl
143 
        print item.cod
144 
        print  "=========================="
16518 kshitij.so 145 
        try:
146 
            message+="""<tr>
147 
            <td style="text-align:center">"""+(item.identifier)+"""</td>
148 
            <td style="text-align:center">"""+(item.category)+"""</td>
149 
            <td style="text-align:center">"""+str(item.rank)+"""</td>
150 
            <td style="text-align:center">"""+str(item.available_price)+"""</td>
151 
            <td style="text-align:center">"""+str(item.gross_price)+"""</td>
152 
            <td style="text-align:center">"""+str(item.in_stock)+"""</td>
153 
            <td style="text-align:center">"""+str(item.coupon)+"""</td>
154 
            <td style="text-align:center">"""+str(item.thumbnail)+"""</td>
155 
            <td style="text-align:center">"""+str(item.source_product_name)+"""</td>
156 
            <td style="text-align:center">"""+str(item.marketPlaceUrl)+"""</td>
157 
            <td style="text-align:center">"""+str(item.cod)+"""</td>
158 
            </tr>"""
159 
        except:
160 
            continue
16381 amit.gupta 161 
    message+="""</tbody></table></body></html>"""
162 
    print message
163 
    #recipients = ['amit.gupta@saholic.com']
164 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com','amit.gupta@saholic.com']
165 
    msg = MIMEMultipart()
16464 kshitij.so 166 
    msg['Subject'] = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 167 
    msg['From'] = ""
168 
    msg['To'] = ",".join(recipients)
16464 kshitij.so 169 
    msg.preamble = "Paytm Best Sellers" + ' - ' + str(datetime.now())
16381 amit.gupta 170 
    html_msg = MIMEText(message, 'html')
171 
    msg.attach(html_msg)
172 
 
173 
    smtpServer = smtplib.SMTP('localhost')
174 
    smtpServer.set_debuglevel(1)
175 
    sender = 'dtr@shop2020.in'
176 
    try:
177 
        smtpServer.sendmail(sender, recipients, msg.as_string())
178 
        print "Successfully sent email"
179 
    except:
180 
        print "Error: unable to send email."
16512 kshitij.so 181 
 
182 
def addExtraAttributes():
183 
    for e in exceptionList:
184 
        url = "https://catalog.paytm.com/v1/mobile/product/%s" %(e.identifier.strip())
185 
        try:
186 
            response_data = fetchResponseUsingProxy(url,  proxy=False)
187 
        except:
188 
            continue
189 
        input_json = json.loads(response_data)
190 
        offerPrice = float(input_json['offer_price'])
191 
        e.cod = int(input_json['pay_type_supported'].get('COD'))
192 
        offerUrl = (input_json['offer_url'])
193 
        e.in_stock = (input_json['instock'])
194 
        try:
195 
            offers = fetchOffers(offerUrl)
196 
        except:
197 
            continue
198 
        bestOffer = {}
16514 kshitij.so 199 
        e.gross_price = float(offerPrice)
16512 kshitij.so 200 
        effective_price = offerPrice
201 
        coupon = ""
202 
        for offer_data in offers.get('codes'):
203 
            if effective_price > offer_data.get('effective_price'):
204 
                effective_price = offer_data.get('effective_price')
205 
                bestOffer = offer_data
206 
                coupon = bestOffer.get('code')
207 
        e.source_product_name = input_json['name']
208 
        e.thumbnail = input_json['thumbnail']
209 
        e.marketPlaceUrl = input_json['shareurl']
210 
        e.coupon = coupon
16514 kshitij.so 211 
        e.available_price = float(effective_price)
16512 kshitij.so 212 
 
16381 amit.gupta 213 
 
214 
 
215 
 
216 
def main():
217 
    scrapeBestSellerMobiles()
218 
    if len(bestSellers) > 0:
16464 kshitij.so 219 
        resetRanks(3)
16381 amit.gupta 220 
        commitBestSellers("MOBILE")
221 
    scrapeBestSellerTablets()
222 
    if len(bestSellers) > 0:
223 
        resetRanks(5)
224 
        commitBestSellers("TABLET")
16512 kshitij.so 225 
    addExtraAttributes()
16381 amit.gupta 226 
    sendEmail()
227 
 
228 
if __name__=='__main__':
229 
    main()