Subversion Repositories SmartDukaan

Rev

Rev 19240 | Rev 19647 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 19240 Rev 19645
Line 1... Line -...
1 
import urllib2
 - 
 
2 
from BeautifulSoup import BeautifulSoup
 - 
 
3 
import pymongo
 - 
 
4 
import re
 - 
 
5 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
 1 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy, get_mongo_connection, CATEGORY_MAP
6 
from datetime import datetime
 2 
from datetime import datetime
7 
import optparse
 3 
import optparse
8 
import smtplib
 4 
import smtplib
9 
from email.mime.text import MIMEText
 5 
from email.mime.text import MIMEText
10 
from email.mime.multipart import MIMEMultipart
 6 
from email.mime.multipart import MIMEMultipart
- 
 
 7 
import json
- 
 
 8 
 
- 
 
 9 
 
- 
 
 10 
 
- 
 
 11 
headers = {
- 
 
 12 
           'Browser-Name': 'Chrome',
- 
 
 13 
           'User-Agent': 'Mozilla/5.0 (Linux; Android 5.1.1; A0001 Build/LMY48B; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/43.0.2357.121 Mobile Safari/537.36 FKUA/Retail/550900/Android/Mobile (OnePlus/A0001)',
11 
from dtr.utils.utils import fetchResponseUsingProxy
 14 
           'Host': 'mobileapi.flipkart.net'
- 
 
 15 
        }
12 
 
 16 
 
13 
con = None
 17 
con = None
14 
parser = optparse.OptionParser()
 18 
parser = optparse.OptionParser()
15 
parser.add_option("-m", "--m", dest="mongoHost",
 19 
parser.add_option("-m", "--m", dest="mongoHost",
16 
                      default="localhost",
 20 
                      default="localhost",
Line 21... Line 25...
21 
 
 25 
 
22 
bestSellers = []
 26 
bestSellers = []
23 
now = datetime.now()
 27 
now = datetime.now()
24 
exceptionList = []
 28 
exceptionList = []
25 
 
 29 
 
- 
 
 30 
 
- 
 
 31 
 
26 
class __RankInfo:
 32 
class __RankInfo:
27
33
28 
    def __init__(self, identifier, rank, category):
 34 
    def __init__(self, identifier, rank, category, title, url):
29 
        self.identifier = identifier
 35 
        self.identifier = identifier
30 
        self.rank  = rank
 36 
        self.rank  = rank
31 
        self.category = category
 37 
        self.category = category
- 
 
 38 
        self.title = title
- 
 
 39 
        self.url = url
32 
 
 40 
 
33 
def get_mongo_connection(host=options.mongoHost, port=27017):
 - 
 
34 
    global con
 - 
 
35 
    if con is None:
 - 
 
36 
        print "Establishing connection %s host and port %d" %(host,port)
 - 
 
37 
        try:
 - 
 
38 
            con = pymongo.MongoClient(host, port)
 - 
 
39 
        except Exception, e:
 - 
 
40 
 
 - 
 
41 
            print e
 - 
 
42 
            return None
 - 
 
43 
    return con
 - 
 
44 
 
 - 
 
45 
def getSoupObject(url):
 - 
 
46 
    print "Getting soup object for"
 - 
 
47 
    print url
 - 
 
48 
    soup = None
 - 
 
49 
    response_data = fetchResponseUsingProxy(url,proxy=False)
 - 
 
50 
    print response_data
 - 
 
51 
    try:
 - 
 
52 
        page=response_data.decode("utf-8")
 - 
 
53 
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
 - 
 
54 
    except:
 - 
 
55 
        soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
 - 
 
56 
    return soup
 - 
 
57 
 
 41 
 
58 
def scrapeBestSellerMobiles():
 42 
def scrapeBestSellerMobiles():
59 
    global bestSellers
 43 
    global bestSellers
60 
    rank = 0
 44 
    rank = 1
61 
    for i in [1, 21, 41, 61, 81]:
 45 
    for i in range(0,100,10):
62 
        url = "http://www.flipkart.com/lc/pr/pv1/spotList1/spot1/productList?sid=tyy,4io&filterNone=true&start=%d&ajax=true" %(i)
 46 
        url = "http://mobileapi.flipkart.net/3/discover/getSearch?store=tyy/4io&start=%d&count=10"%(i)
63
- 
 
64
47 
        print url
65 
        soup = getSoupObject(url)
 - 
 
66 
        product_divs = soup.findAll('div',{'class':re.compile('.*browse-product')})
 48 
        response_data = fetchResponseUsingProxy(url, headers, livePricing=None, proxy=False, flipkart=False)
67 
        for x in product_divs:
 49 
        input_json = json.loads(response_data)
68 
            rank  = rank +1
 50 
        for identifier, data in (input_json['RESPONSE']['product']).iteritems():
69 
            print rank,
 - 
 
70 
            print '\t',
 51 
            title = data['value']['titles']['title'] + " "+data['value']['titles']['subtitle']
71 
            print x['data-pid']
 52 
            url =  (data['value']['smartUrl']).replace("http://dl.flipkart.com/dl", "http://www.flipkart.com")
72 
            r_info = __RankInfo(x['data-pid'].strip(),rank, None)
 53 
            r_info = __RankInfo(identifier, rank, 3, title, url)
73 
            bestSellers.append(r_info)
 54 
            bestSellers.append(r_info)
- 
 
 55 
            rank = rank + 1
- 
 
 56
74 
 
 57 
 
75 
def scrapeBestSellerTablets():
 58 
def scrapeBestSellerTablets():
76 
    global bestSellers
 59 
    global bestSellers
77 
    bestSellers = []
 - 
 
78 
    rank = 0
 60 
    rank = 1
79 
    for i in [1, 21, 41, 61, 81]:
 61 
    for i in range(0,100,10):
80 
        url = "http://www.flipkart.com/lc/pr/pv1/spotList1/spot1/productList?sid=tyy,hry&filterNone=true&start=%d&ajax=true" %(i)
 62 
        url = "http://mobileapi.flipkart.net/3/discover/getSearch?store=tyy/hry&start=%d&count=10"%(i)
81 
        soup = getSoupObject(url)
 63 
        print url
82 
        product_divs = soup.findAll('div',{'class':re.compile('.*browse-product')})
 64 
        response_data = fetchResponseUsingProxy(url, headers, livePricing=None, proxy=False, flipkart=False)
83 
        for x in product_divs:
 65 
        input_json = json.loads(response_data)
84 
            rank  = rank +1
 66 
        for identifier, data in (input_json['RESPONSE']['product']).iteritems():
85 
            print rank,
 - 
 
86 
            print '\t',
 67 
            title = data['value']['titles']['title'] + " "+data['value']['titles']['subtitle']
87 
            print x['data-pid']
 68 
            url =  (data['value']['smartUrl']).replace("http://dl.flipkart.com/dl", "http://www.flipkart.com")
88 
            r_info = __RankInfo(x['data-pid'].strip(),rank, None)
 69 
            r_info = __RankInfo(identifier, rank, 5, title, url)
89 
            bestSellers.append(r_info)
 70 
            bestSellers.append(r_info)
- 
 
 71 
            rank = rank + 1
90 
 
 72 
 
91 
def commitBestSellers(category):
 73 
def commitBestSellers(category):
92 
    global exceptionList
 74 
    global exceptionList
93 
    for x in bestSellers:
 75 
    for x in bestSellers:
94 
        print x.rank,
 - 
 
95 
        print '\t',
 - 
 
96 
        print x.identifier,
 - 
 
97 
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier.strip()})
 76 
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier.strip(),'category_id':category})
98 
        print "count sku",
 - 
 
99 
        print '\t',
 - 
 
100 
        if len(list(col)) == 0:
 77 
        if len(list(col)) == 0:
101 
            x.category = category
 78 
            x.category = category
102 
            exceptionList.append(x)
 79 
            exceptionList.append(x)
103 
        else:
 80 
        else:
104 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier.strip()}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
 81 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier.strip(),'category_id':category}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
105 
 
 82 
 
106 
def resetRanks(category):
 83 
def resetRanks(category):
107 
    oldRankedItems = get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':2,'category':category})
 - 
 
108 
    for item in oldRankedItems:
 - 
 
109 
        get_mongo_connection().Catalog.MasterData.update({'_id':item['_id']}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
 84 
    get_mongo_connection().Catalog.MasterData.update({'category_id':category}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
110 
 
 85 
 
111 
def sendEmail():
 86 
def sendEmail():
112 
    message="""<html>
 87 
    message="""<html>
113 
            <body>
 88 
            <body>
114 
            <h3>Flipkart Best Sellers not in master</h3>
 89 
            <h3>Flipkart Best Sellers not in master</h3>
115 
            <table border="1" style="width:100%;">
 90 
            <table border="1" style="width:100%;">
116 
            <thead>
 91 
            <thead>
117 
            <tr><th>Identifier</th>
 92 
            <tr><th>Identifier</th>
- 
 
 93 
            <th>Title</th>
118 
            <th>Category</th>
 94 
            <th>Category</th>
119 
            <th>Rank</th>
 95 
            <th>Rank</th>
- 
 
 96 
            <th>URL</th>
120 
            </tr></thead>
 97 
            </tr></thead>
121 
            <tbody>"""
 98 
            <tbody>"""
122 
    for item in exceptionList:
 99 
    for item in exceptionList:
123 
        message+="""<tr>
 100 
        message+="""<tr>
124 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
 101 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
125 
        <td style="text-align:center">"""+(item.category)+"""</td>
 102 
        <td style="text-align:center">"""+(item.title)+"""</td>
- 
 
 103 
        <td style="text-align:center">"""+(CATEGORY_MAP.get(item.category))+"""</td>
126 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
 104 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
- 
 
 105 
        <td style="text-align:center">"""+(item.url)+"""</td>
127 
        </tr>"""
 106 
        </tr>"""
128 
    message+="""</tbody></table></body></html>"""
 107 
    message+="""</tbody></table></body></html>"""
129 
    print message
 - 
 
130 
    recipients = ['kshitij.sood@saholic.com']
 108 
    #recipients = ['kshitij.sood@saholic.com']
131 
    #recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
 109 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
132 
    msg = MIMEMultipart()
 110 
    msg = MIMEMultipart()
133 
    msg['Subject'] = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
 111 
    msg['Subject'] = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
134 
    msg['From'] = ""
 112 
    msg['From'] = ""
135 
    msg['To'] = ",".join(recipients)
 113 
    msg['To'] = ",".join(recipients)
136 
    msg.preamble = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
 114 
    msg.preamble = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
Line 145... Line 123...
145 
        print "Successfully sent email"
 123 
        print "Successfully sent email"
146 
    except:
 124 
    except:
147 
        print "Error: unable to send email."
 125 
        print "Error: unable to send email."
148
126
149 
def main():
 127 
def main():
- 
 
 128 
    import time
150 
    scrapeBestSellerMobiles()
 129 
    scrapeBestSellerMobiles()
151 
    if len(bestSellers) > 0:
 130 
    if len(bestSellers) > 0:
152 
        resetRanks('Mobiles')
 131 
        resetRanks(3)
- 
 
 132 
        time.sleep(20)
153 
        commitBestSellers('MOBILES')
 133 
        commitBestSellers(3)
154 
    scrapeBestSellerTablets()
 134 
    scrapeBestSellerTablets()
155 
    if len(bestSellers) > 0:
 135 
    if len(bestSellers) > 0:
156 
        resetRanks('Tablets')
 136 
        resetRanks(5)
- 
 
 137 
        time.sleep(20)
157 
        commitBestSellers('TABLETS')
 138 
        commitBestSellers(5)
158 
    sendEmail()
 139 
    sendEmail()
159
140
160 
if __name__=='__main__':
 141 
if __name__=='__main__':
161 
    main()
 142 
    main()
162 

            143