Subversion Repositories SmartDukaan

Rev

Rev 20358 | Rev 22736 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
19645 kshitij.so 1 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy, get_mongo_connection, CATEGORY_MAP
13828 kshitij.so 2 
from datetime import datetime
14257 kshitij.so 3 
import optparse
14379 kshitij.so 4 
import smtplib
5 
from email.mime.text import MIMEText
6 
from email.mime.multipart import MIMEMultipart
19645 kshitij.so 7 
import json
20357 kshitij.so 8 
import chardet
21135 kshitij.so 9 
from shop2020.utils.EmailAttachmentSender import get_attachment_part
10 
from shop2020.utils import EmailAttachmentSender
13754 kshitij.so 11 
 
19645 kshitij.so 12 
 
13 
headers = {
14 
           'Browser-Name': 'Chrome',
15 
           'User-Agent': 'Mozilla/5.0 (Linux; Android 5.1.1; A0001 Build/LMY48B; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/43.0.2357.121 Mobile Safari/537.36 FKUA/Retail/550900/Android/Mobile (OnePlus/A0001)',
16 
           'Host': 'mobileapi.flipkart.net'
17 
        }
18 
 
13754 kshitij.so 19 
con = None
14257 kshitij.so 20 
parser = optparse.OptionParser()
21 
parser.add_option("-m", "--m", dest="mongoHost",
22 
                      default="localhost",
23 
                      type="string", help="The HOST where the mongo server is running",
24 
                      metavar="mongo_host")
25 
 
26 
(options, args) = parser.parse_args()
27 
 
13754 kshitij.so 28 
bestSellers = []
13828 kshitij.so 29 
now = datetime.now()
14379 kshitij.so 30 
exceptionList = []
13754 kshitij.so 31 
 
20357 kshitij.so 32 
xstr = lambda s: s or ""
19645 kshitij.so 33 
 
13754 kshitij.so 34 
class __RankInfo:
35 
 
19645 kshitij.so 36 
    def __init__(self, identifier, rank, category, title, url):
13754 kshitij.so 37 
        self.identifier = identifier
38 
        self.rank  = rank
14379 kshitij.so 39 
        self.category = category
19645 kshitij.so 40 
        self.title = title
41 
        self.url = url
13754 kshitij.so 42 
 
18284 kshitij.so 43 
 
13754 kshitij.so 44 
def scrapeBestSellerMobiles():
45 
    global bestSellers
19645 kshitij.so 46 
    rank = 1
47 
    for i in range(0,100,10):
48 
        url = "http://mobileapi.flipkart.net/3/discover/getSearch?store=tyy/4io&start=%d&count=10"%(i)
49 
        print url
50 
        response_data = fetchResponseUsingProxy(url, headers, livePricing=None, proxy=False, flipkart=False)
51 
        input_json = json.loads(response_data)
52 
        for identifier, data in (input_json['RESPONSE']['product']).iteritems():
20357 kshitij.so 53 
            t_title = data['value']['titles']['title']
54 
            t_subtitle = data['value']['titles']['subtitle']
55 
            title = (xstr(t_title) + " " + xstr(t_subtitle)).strip()
56 
            print title
19645 kshitij.so 57 
            url =  (data['value']['smartUrl']).replace("http://dl.flipkart.com/dl", "http://www.flipkart.com")
58 
            r_info = __RankInfo(identifier, rank, 3, title, url)
13754 kshitij.so 59 
            bestSellers.append(r_info)
19645 kshitij.so 60 
            rank = rank + 1
61 
 
13754 kshitij.so 62 
 
63 
def scrapeBestSellerTablets():
64 
    global bestSellers
19645 kshitij.so 65 
    rank = 1
66 
    for i in range(0,100,10):
67 
        url = "http://mobileapi.flipkart.net/3/discover/getSearch?store=tyy/hry&start=%d&count=10"%(i)
68 
        print url
69 
        response_data = fetchResponseUsingProxy(url, headers, livePricing=None, proxy=False, flipkart=False)
70 
        input_json = json.loads(response_data)
71 
        for identifier, data in (input_json['RESPONSE']['product']).iteritems():
20357 kshitij.so 72 
            t_title = data['value']['titles']['title']
73 
            t_subtitle = data['value']['titles']['subtitle']
74 
            title = (xstr(t_title) + " " + xstr(t_subtitle)).strip()
75 
            print title
19645 kshitij.so 76 
            url =  (data['value']['smartUrl']).replace("http://dl.flipkart.com/dl", "http://www.flipkart.com")
20358 kshitij.so 77 
            print url
78 
            print identifier
79 
            print "=================="
19645 kshitij.so 80 
            r_info = __RankInfo(identifier, rank, 5, title, url)
13754 kshitij.so 81 
            bestSellers.append(r_info)
19645 kshitij.so 82 
            rank = rank + 1
13754 kshitij.so 83 
 
19648 kshitij.so 84 
def commitBestSellers():
14379 kshitij.so 85 
    global exceptionList
13754 kshitij.so 86 
    for x in bestSellers:
19648 kshitij.so 87 
        col = get_mongo_connection(host=options.mongoHost).Catalog.MasterData.find({'identifier':x.identifier.strip()})
14379 kshitij.so 88 
        if len(list(col)) == 0:
89 
            exceptionList.append(x)
90 
        else:
19648 kshitij.so 91 
            get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'identifier':x.identifier.strip()}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
13754 kshitij.so 92 
 
93 
def resetRanks(category):
19647 kshitij.so 94 
    get_mongo_connection(host=options.mongoHost).Catalog.MasterData.update({'category_id':category}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
14379 kshitij.so 95 
 
96 
def sendEmail():
97 
    message="""<html>
98 
            <body>
99 
            <h3>Flipkart Best Sellers not in master</h3>
100 
            <table border="1" style="width:100%;">
101 
            <thead>
102 
            <tr><th>Identifier</th>
19645 kshitij.so 103 
            <th>Title</th>
14379 kshitij.so 104 
            <th>Category</th>
105 
            <th>Rank</th>
19645 kshitij.so 106 
            <th>URL</th>
14379 kshitij.so 107 
            </tr></thead>
108 
            <tbody>"""
109 
    for item in exceptionList:
110 
        message+="""<tr>
111 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
19645 kshitij.so 112 
        <td style="text-align:center">"""+(item.title)+"""</td>
113 
        <td style="text-align:center">"""+(CATEGORY_MAP.get(item.category))+"""</td>
14379 kshitij.so 114 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
19645 kshitij.so 115 
        <td style="text-align:center">"""+(item.url)+"""</td>
14379 kshitij.so 116 
        </tr>"""
117 
    message+="""</tbody></table></body></html>"""
20358 kshitij.so 118 
    message = message.decode('ascii', 'ignore')
19645 kshitij.so 119 
    #recipients = ['kshitij.sood@saholic.com']
21135 kshitij.so 120 
    recipients = ['kshitij.sood@saholic.com','ritesh.chauhan@saholic.com','aishwarya.singh@saholic.com']
121 
    EmailAttachmentSender.mail_send_grid("dtr@profitmandi.com","apikey", "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", recipients, "Flipkart Best Sellers",message ,[],[],[])
122 
 
14379 kshitij.so 123 
 
13754 kshitij.so 124 
def main():
125 
    scrapeBestSellerMobiles()
19648 kshitij.so 126 
    resetRanks(3)
13754 kshitij.so 127 
    scrapeBestSellerTablets()
19648 kshitij.so 128 
    resetRanks(5)
129 
    commitBestSellers()
14379 kshitij.so 130 
    sendEmail()
13754 kshitij.so 131 
 
132 
if __name__=='__main__':
133 
    main()