Subversion Repositories SmartDukaan

Rev

Rev 14257 | Rev 20172 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14257 Rev 14379
Line 4... Line 4...
4 
import pymongo
 4 
import pymongo
5 
import re
 5 
import re
6 
from dtr.utils.utils import to_java_date
 6 
from dtr.utils.utils import to_java_date
7 
from datetime import datetime
 7 
from datetime import datetime
8 
import optparse
 8 
import optparse
- 
 
 9 
import smtplib
- 
 
 10 
from email.mime.text import MIMEText
- 
 
 11 
from email.mime.multipart import MIMEMultipart
9 
 
 12 
 
10 
con = None
 13 
con = None
11 
parser = optparse.OptionParser()
 14 
parser = optparse.OptionParser()
12 
parser.add_option("-m", "--m", dest="mongoHost",
 15 
parser.add_option("-m", "--m", dest="mongoHost",
13 
                      default="localhost",
 16 
                      default="localhost",
Line 19... Line 22...
19 
bestSellers = []
 22 
bestSellers = []
20 
now = datetime.now()
 23 
now = datetime.now()
21 
mobUrl = "http://www.saholic.com/mobile-phone/10006"
 24 
mobUrl = "http://www.saholic.com/mobile-phone/10006"
22 
tabUrl = "http://www.saholic.com/all-tablets/10010"
 25 
tabUrl = "http://www.saholic.com/all-tablets/10010"
23 
 
 26 
 
- 
 
 27 
exceptionList = []
- 
 
 28 
 
24 
class __RankInfo:
 29 
class __RankInfo:
25
30
26 
    def __init__(self, identifier, rank):
 31 
    def __init__(self, identifier, rank, category):
27 
        self.identifier = identifier
 32 
        self.identifier = identifier
28 
        self.rank  = rank
 33 
        self.rank  = rank
- 
 
 34 
        self.category = category
29 
 
 35 
 
30 
def get_mongo_connection(host=options.mongoHost, port=27017):
 36 
def get_mongo_connection(host=options.mongoHost, port=27017):
31 
    global con
 37 
    global con
32 
    if con is None:
 38 
    if con is None:
33 
        print "Establishing connection %s host and port %d" %(host,port)
 39 
        print "Establishing connection %s host and port %d" %(host,port)
Line 76... Line 82...
76 
        soup = getSoupObject(mobUrl,data)
 82 
        soup = getSoupObject(mobUrl,data)
77 
        titleTags = soup.findAll('div',{'class':'title'})
 83 
        titleTags = soup.findAll('div',{'class':'title'})
78 
        for titleTag in titleTags:
 84 
        for titleTag in titleTags:
79 
            print titleTag.find('a').text
 85 
            print titleTag.find('a').text
80 
            identifier = getSaholicIdentifier(titleTag.find('a')['href'])
 86 
            identifier = getSaholicIdentifier(titleTag.find('a')['href'])
81 
            r_info = __RankInfo(str(identifier), rank)
 87 
            r_info = __RankInfo(str(identifier), rank ,None)
82 
            bestSellers.append(r_info)
 88 
            bestSellers.append(r_info)
83 
            rank = rank + 1
 89 
            rank = rank + 1
84 
 
 90 
 
85 
def scrapeBestSellerTablets():
 91 
def scrapeBestSellerTablets():
86 
    global bestSellers
 92 
    global bestSellers
Line 92... Line 98...
92 
        soup = getSoupObject(tabUrl, data)
 98 
        soup = getSoupObject(tabUrl, data)
93 
        titleTags = soup.findAll('div',{'class':'title'})
 99 
        titleTags = soup.findAll('div',{'class':'title'})
94 
        for titleTag in titleTags:
 100 
        for titleTag in titleTags:
95 
            print titleTag.find('a').text
 101 
            print titleTag.find('a').text
96 
            identifier = getSaholicIdentifier(titleTag.find('a')['href'])
 102 
            identifier = getSaholicIdentifier(titleTag.find('a')['href'])
97 
            r_info = __RankInfo(str(identifier), rank)
 103 
            r_info = __RankInfo(str(identifier), rank, None)
98 
            bestSellers.append(r_info)
 104 
            bestSellers.append(r_info)
99 
            rank = rank + 1
 105 
            rank = rank + 1
100 
 
 106 
 
101 
def getSaholicIdentifier(url):
 107 
def getSaholicIdentifier(url):
102 
    return url[url.rfind('-')+len('-'):]
 108 
    return url[url.rfind('-')+len('-'):]
Line 105... Line 111...
105 
    oldRankedItems = get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':4,'category':category})
 111 
    oldRankedItems = get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':4,'category':category})
106 
    for item in oldRankedItems:
 112 
    for item in oldRankedItems:
107 
        print item['_id']
 113 
        print item['_id']
108 
        get_mongo_connection().Catalog.MasterData.update({'_id':item['_id']}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
 114 
        get_mongo_connection().Catalog.MasterData.update({'_id':item['_id']}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
109 
 
 115 
 
110 
def commitBestSellers():
 116 
def commitBestSellers(category):
- 
 
 117 
    global exceptionList
111 
    print "Rank",
 118 
    print "Rank",
112 
    print '\t',
 119 
    print '\t',
113 
    print 'Identifier'
 120 
    print 'Identifier'
114 
    for x in bestSellers:
 121 
    for x in bestSellers:
115 
        print x.rank,
 122 
        print x.rank,
116 
        print '\t',
 123 
        print '\t',
117 
        print x.identifier,
 124 
        print x.identifier,
118 
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier})
 125 
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier})
119 
        print "count sku",
 126 
        print "count sku",
120 
        print '\t',
 127 
        print '\t',
121 
        print len(list(col))
 128 
        if len(list(col)) == 0:
- 
 
 129 
            x.category = category
- 
 
 130 
            exceptionList.append(x)
- 
 
 131 
        else:
122 
        get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier,'source_id':4}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
 132 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier,'source_id':4}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
123 
 
 133 
 
- 
 
 134 
def sendEmail():
- 
 
 135 
    message="""<html>
- 
 
 136 
            <body>
- 
 
 137 
            <h3>Saholic Best Sellers not in master</h3>
- 
 
 138 
            <table border="1" style="width:100%;">
- 
 
 139 
            <thead>
- 
 
 140 
            <tr><th>Identifier</th>
- 
 
 141 
            <th>Category</th>
- 
 
 142 
            <th>Rank</th>
- 
 
 143 
            </tr></thead>
- 
 
 144 
            <tbody>"""
- 
 
 145 
    for item in exceptionList:
- 
 
 146 
        message+="""<tr>
- 
 
 147 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
- 
 
 148 
        <td style="text-align:center">"""+(item.category)+"""</td>
- 
 
 149 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
- 
 
 150 
        </tr>"""
- 
 
 151 
    message+="""</tbody></table></body></html>"""
- 
 
 152 
    print message
- 
 
 153 
    #recipients = ['kshitij.sood@saholic.com']
- 
 
 154 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
- 
 
 155 
    msg = MIMEMultipart()
- 
 
 156 
    msg['Subject'] = "Saholic Best Sellers" + ' - ' + str(datetime.now())
- 
 
 157 
    msg['From'] = ""
- 
 
 158 
    msg['To'] = ",".join(recipients)
- 
 
 159 
    msg.preamble = "Saholic Best Sellers" + ' - ' + str(datetime.now())
- 
 
 160 
    html_msg = MIMEText(message, 'html')
- 
 
 161 
    msg.attach(html_msg)
- 
 
 162
- 
 
 163 
    smtpServer = smtplib.SMTP('localhost')
- 
 
 164 
    smtpServer.set_debuglevel(1)
- 
 
 165 
    sender = 'dtr@shop2020.in'
124
166 
    try:
- 
 
 167 
        smtpServer.sendmail(sender, recipients, msg.as_string())
- 
 
 168 
        print "Successfully sent email"
- 
 
 169 
    except:
- 
 
 170 
        print "Error: unable to send email."
125 
 
 171 
 
126 
def main():
 172 
def main():
127 
    scrapeBestSellerMobiles()
 173 
    scrapeBestSellerMobiles()
128 
    if len(bestSellers) > 0:
 174 
    if len(bestSellers) > 0:
129 
        resetRanks('Mobiles')
 175 
        resetRanks('Mobiles')
130 
        commitBestSellers()
 176 
        commitBestSellers('MOBILES')
131 
    scrapeBestSellerTablets()
 177 
    scrapeBestSellerTablets()
132 
    if len(bestSellers) > 0:
 178 
    if len(bestSellers) > 0:
133 
        resetRanks('Tablets')
 179 
        resetRanks('Tablets')
134 
        commitBestSellers()
 180 
        commitBestSellers('TABLETS')
- 
 
 181
- 
 
 182 
    sendEmail()
- 
 
 183
135 
if __name__=='__main__':
 184 
if __name__=='__main__':
136 
    main()
 185 
    main()
137 

            186