Subversion Repositories SmartDukaan

Rev

Rev 20357 | Rev 21135 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
19645 kshitij.so 1 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy, get_mongo_connection, CATEGORY_MAP
13828 kshitij.so 2 
from datetime import datetime
14257 kshitij.so 3 
import optparse
14379 kshitij.so 4 
import smtplib
5 
from email.mime.text import MIMEText
6 
from email.mime.multipart import MIMEMultipart
19645 kshitij.so 7 
import json
20357 kshitij.so 8 
import chardet
13754 kshitij.so 9 
 
19645 kshitij.so 10 
 
11 
 
12 
headers = {
13 
           'Browser-Name': 'Chrome',
14 
           'User-Agent': 'Mozilla/5.0 (Linux; Android 5.1.1; A0001 Build/LMY48B; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/43.0.2357.121 Mobile Safari/537.36 FKUA/Retail/550900/Android/Mobile (OnePlus/A0001)',
15 
           'Host': 'mobileapi.flipkart.net'
16 
        }
17 
 
13754 kshitij.so 18 
con = None
14257 kshitij.so 19 
parser = optparse.OptionParser()
20 
parser.add_option("-m", "--m", dest="mongoHost",
21 
                      default="localhost",
22 
                      type="string", help="The HOST where the mongo server is running",
23 
                      metavar="mongo_host")
24 
 
25 
(options, args) = parser.parse_args()
26 
 
13754 kshitij.so 27 
bestSellers = []
13828 kshitij.so 28 
now = datetime.now()
14379 kshitij.so 29 
exceptionList = []
13754 kshitij.so 30 
 
20357 kshitij.so 31 
xstr = lambda s: s or ""
19645 kshitij.so 32 
 
13754 kshitij.so 33 
class __RankInfo:
34 
 
19645 kshitij.so 35 
    def __init__(self, identifier, rank, category, title, url):
13754 kshitij.so 36 
        self.identifier = identifier
37 
        self.rank  = rank
14379 kshitij.so 38 
        self.category = category
19645 kshitij.so 39 
        self.title = title
40 
        self.url = url
13754 kshitij.so 41 
 
18284 kshitij.so 42 
 
13754 kshitij.so 43 
def scrapeBestSellerMobiles():
44 
    global bestSellers
19645 kshitij.so 45 
    rank = 1
46 
    for i in range(0,100,10):
47 
        url = "http://mobileapi.flipkart.net/3/discover/getSearch?store=tyy/4io&start=%d&count=10"%(i)
48 
        print url
49 
        response_data = fetchResponseUsingProxy(url, headers, livePricing=None, proxy=False, flipkart=False)
50 
        input_json = json.loads(response_data)
51 
        for identifier, data in (input_json['RESPONSE']['product']).iteritems():
20357 kshitij.so 52 
            t_title = data['value']['titles']['title']
53 
            t_subtitle = data['value']['titles']['subtitle']
54 
            title = (xstr(t_title) + " " + xstr(t_subtitle)).strip()
55 
            print title
19645 kshitij.so 56 
            url =  (data['value']['smartUrl']).replace("http://dl.flipkart.com/dl", "http://www.flipkart.com")
57 
            r_info = __RankInfo(identifier, rank, 3, title, url)
13754 kshitij.so 58 
            bestSellers.append(r_info)
19645 kshitij.so 59 
            rank = rank + 1
60 
 
13754 kshitij.so 61 
 
62 
def scrapeBestSellerTablets():
63 
    global bestSellers
19645 kshitij.so 64 
    rank = 1
65 
    for i in range(0,100,10):
66 
        url = "http://mobileapi.flipkart.net/3/discover/getSearch?store=tyy/hry&start=%d&count=10"%(i)
67 
        print url
68 
        response_data = fetchResponseUsingProxy(url, headers, livePricing=None, proxy=False, flipkart=False)
69 
        input_json = json.loads(response_data)
70 
        for identifier, data in (input_json['RESPONSE']['product']).iteritems():
20357 kshitij.so 71 
            t_title = data['value']['titles']['title']
72 
            t_subtitle = data['value']['titles']['subtitle']
73 
            title = (xstr(t_title) + " " + xstr(t_subtitle)).strip()
74 
            print title
19645 kshitij.so 75 
            url =  (data['value']['smartUrl']).replace("http://dl.flipkart.com/dl", "http://www.flipkart.com")
20358 kshitij.so 76 
            print url
77 
            print identifier
78 
            print "=================="
19645 kshitij.so 79 
            r_info = __RankInfo(identifier, rank, 5, title, url)
13754 kshitij.so 80 
            bestSellers.append(r_info)
19645 kshitij.so 81 
            rank = rank + 1
13754 kshitij.so 82 
 
19648 kshitij.so 83 
def commitBestSellers():
14379 kshitij.so 84 
    global exceptionList
13754 kshitij.so 85 
    for x in bestSellers:
19648 kshitij.so 86 
        col = get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'identifier':x.identifier.strip()})
14379 kshitij.so 87 
        if len(list(col)) == 0:
88 
            exceptionList.append(x)
89 
        else:
19648 kshitij.so 90 
            get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'identifier':x.identifier.strip()}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
13754 kshitij.so 91 
 
92 
def resetRanks(category):
19647 kshitij.so 93 
    get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'category_id':category}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
14379 kshitij.so 94 
 
95 
def sendEmail():
96 
    message="""<html>
97 
            <body>
98 
            <h3>Flipkart Best Sellers not in master</h3>
99 
            <table border="1" style="width:100%;">
100 
            <thead>
101 
            <tr><th>Identifier</th>
19645 kshitij.so 102 
            <th>Title</th>
14379 kshitij.so 103 
            <th>Category</th>
104 
            <th>Rank</th>
19645 kshitij.so 105 
            <th>URL</th>
14379 kshitij.so 106 
            </tr></thead>
107 
            <tbody>"""
108 
    for item in exceptionList:
109 
        message+="""<tr>
110 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
19645 kshitij.so 111 
        <td style="text-align:center">"""+(item.title)+"""</td>
112 
        <td style="text-align:center">"""+(CATEGORY_MAP.get(item.category))+"""</td>
14379 kshitij.so 113 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
19645 kshitij.so 114 
        <td style="text-align:center">"""+(item.url)+"""</td>
14379 kshitij.so 115 
        </tr>"""
116 
    message+="""</tbody></table></body></html>"""
20358 kshitij.so 117 
    message = message.decode('ascii', 'ignore')
19645 kshitij.so 118 
    #recipients = ['kshitij.sood@saholic.com']
20172 aman.kumar 119 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','ritesh.chauhan@saholic.com','khushal.bhatia@saholic.com']
14379 kshitij.so 120 
    msg = MIMEMultipart()
121 
    msg['Subject'] = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
122 
    msg['From'] = ""
123 
    msg['To'] = ",".join(recipients)
124 
    msg.preamble = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
125 
    html_msg = MIMEText(message, 'html')
126 
    msg.attach(html_msg)
127 
 
128 
    smtpServer = smtplib.SMTP('localhost')
129 
    smtpServer.set_debuglevel(1)
130 
    sender = 'dtr@shop2020.in'
131 
    try:
132 
        smtpServer.sendmail(sender, recipients, msg.as_string())
133 
        print "Successfully sent email"
134 
    except:
135 
        print "Error: unable to send email."
13754 kshitij.so 136 
 
137 
def main():
138 
    scrapeBestSellerMobiles()
19648 kshitij.so 139 
    resetRanks(3)
13754 kshitij.so 140 
    scrapeBestSellerTablets()
19648 kshitij.so 141 
    resetRanks(5)
142 
    commitBestSellers()
14379 kshitij.so 143 
    sendEmail()
13754 kshitij.so 144 
 
145 
if __name__=='__main__':
146 
    main()