Subversion Repositories SmartDukaan

Rev

Rev 19647 | Rev 20172 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
19645 kshitij.so 1 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy, get_mongo_connection, CATEGORY_MAP
13828 kshitij.so 2 
from datetime import datetime
14257 kshitij.so 3 
import optparse
14379 kshitij.so 4 
import smtplib
5 
from email.mime.text import MIMEText
6 
from email.mime.multipart import MIMEMultipart
19645 kshitij.so 7 
import json
13754 kshitij.so 8 
 
19645 kshitij.so 9 
 
10 
 
11 
headers = {
12 
           'Browser-Name': 'Chrome',
13 
           'User-Agent': 'Mozilla/5.0 (Linux; Android 5.1.1; A0001 Build/LMY48B; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/43.0.2357.121 Mobile Safari/537.36 FKUA/Retail/550900/Android/Mobile (OnePlus/A0001)',
14 
           'Host': 'mobileapi.flipkart.net'
15 
        }
16 
 
13754 kshitij.so 17 
con = None
14257 kshitij.so 18 
parser = optparse.OptionParser()
19 
parser.add_option("-m", "--m", dest="mongoHost",
20 
                      default="localhost",
21 
                      type="string", help="The HOST where the mongo server is running",
22 
                      metavar="mongo_host")
23 
 
24 
(options, args) = parser.parse_args()
25 
 
13754 kshitij.so 26 
bestSellers = []
13828 kshitij.so 27 
now = datetime.now()
14379 kshitij.so 28 
exceptionList = []
13754 kshitij.so 29 
 
19645 kshitij.so 30 
 
31 
 
13754 kshitij.so 32 
class __RankInfo:
33 
 
19645 kshitij.so 34 
    def __init__(self, identifier, rank, category, title, url):
13754 kshitij.so 35 
        self.identifier = identifier
36 
        self.rank  = rank
14379 kshitij.so 37 
        self.category = category
19645 kshitij.so 38 
        self.title = title
39 
        self.url = url
13754 kshitij.so 40 
 
18284 kshitij.so 41 
 
13754 kshitij.so 42 
def scrapeBestSellerMobiles():
43 
    global bestSellers
19645 kshitij.so 44 
    rank = 1
45 
    for i in range(0,100,10):
46 
        url = "http://mobileapi.flipkart.net/3/discover/getSearch?store=tyy/4io&start=%d&count=10"%(i)
47 
        print url
48 
        response_data = fetchResponseUsingProxy(url, headers, livePricing=None, proxy=False, flipkart=False)
49 
        input_json = json.loads(response_data)
50 
        for identifier, data in (input_json['RESPONSE']['product']).iteritems():
51 
            title = data['value']['titles']['title'] + " "+data['value']['titles']['subtitle']
52 
            url =  (data['value']['smartUrl']).replace("http://dl.flipkart.com/dl", "http://www.flipkart.com")
53 
            r_info = __RankInfo(identifier, rank, 3, title, url)
13754 kshitij.so 54 
            bestSellers.append(r_info)
19645 kshitij.so 55 
            rank = rank + 1
56 
 
13754 kshitij.so 57 
 
58 
def scrapeBestSellerTablets():
59 
    global bestSellers
19645 kshitij.so 60 
    rank = 1
61 
    for i in range(0,100,10):
62 
        url = "http://mobileapi.flipkart.net/3/discover/getSearch?store=tyy/hry&start=%d&count=10"%(i)
63 
        print url
64 
        response_data = fetchResponseUsingProxy(url, headers, livePricing=None, proxy=False, flipkart=False)
65 
        input_json = json.loads(response_data)
66 
        for identifier, data in (input_json['RESPONSE']['product']).iteritems():
67 
            title = data['value']['titles']['title'] + " "+data['value']['titles']['subtitle']
68 
            url =  (data['value']['smartUrl']).replace("http://dl.flipkart.com/dl", "http://www.flipkart.com")
69 
            r_info = __RankInfo(identifier, rank, 5, title, url)
13754 kshitij.so 70 
            bestSellers.append(r_info)
19645 kshitij.so 71 
            rank = rank + 1
13754 kshitij.so 72 
 
19648 kshitij.so 73 
def commitBestSellers():
14379 kshitij.so 74 
    global exceptionList
13754 kshitij.so 75 
    for x in bestSellers:
19648 kshitij.so 76 
        col = get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'identifier':x.identifier.strip()})
14379 kshitij.so 77 
        if len(list(col)) == 0:
78 
            exceptionList.append(x)
79 
        else:
19648 kshitij.so 80 
            get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'identifier':x.identifier.strip()}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
13754 kshitij.so 81 
 
82 
def resetRanks(category):
19647 kshitij.so 83 
    get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'category_id':category}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
14379 kshitij.so 84 
 
85 
def sendEmail():
86 
    message="""<html>
87 
            <body>
88 
            <h3>Flipkart Best Sellers not in master</h3>
89 
            <table border="1" style="width:100%;">
90 
            <thead>
91 
            <tr><th>Identifier</th>
19645 kshitij.so 92 
            <th>Title</th>
14379 kshitij.so 93 
            <th>Category</th>
94 
            <th>Rank</th>
19645 kshitij.so 95 
            <th>URL</th>
14379 kshitij.so 96 
            </tr></thead>
97 
            <tbody>"""
98 
    for item in exceptionList:
99 
        message+="""<tr>
100 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
19645 kshitij.so 101 
        <td style="text-align:center">"""+(item.title)+"""</td>
102 
        <td style="text-align:center">"""+(CATEGORY_MAP.get(item.category))+"""</td>
14379 kshitij.so 103 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
19645 kshitij.so 104 
        <td style="text-align:center">"""+(item.url)+"""</td>
14379 kshitij.so 105 
        </tr>"""
106 
    message+="""</tbody></table></body></html>"""
19648 kshitij.so 107 
    print message
19645 kshitij.so 108 
    #recipients = ['kshitij.sood@saholic.com']
109 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
14379 kshitij.so 110 
    msg = MIMEMultipart()
111 
    msg['Subject'] = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
112 
    msg['From'] = ""
113 
    msg['To'] = ",".join(recipients)
114 
    msg.preamble = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
115 
    html_msg = MIMEText(message, 'html')
116 
    msg.attach(html_msg)
117 
 
118 
    smtpServer = smtplib.SMTP('localhost')
119 
    smtpServer.set_debuglevel(1)
120 
    sender = 'dtr@shop2020.in'
121 
    try:
122 
        smtpServer.sendmail(sender, recipients, msg.as_string())
123 
        print "Successfully sent email"
124 
    except:
125 
        print "Error: unable to send email."
13754 kshitij.so 126 
 
127 
def main():
128 
    scrapeBestSellerMobiles()
19648 kshitij.so 129 
    resetRanks(3)
13754 kshitij.so 130 
    scrapeBestSellerTablets()
19648 kshitij.so 131 
    resetRanks(5)
132 
    commitBestSellers()
14379 kshitij.so 133 
    sendEmail()
13754 kshitij.so 134 
 
135 
if __name__=='__main__':
136 
    main()