Subversion Repositories SmartDukaan

Rev

Rev 15088 | Rev 17390 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 15088 Rev 15829
Line 5... Line 5...
5 
from datetime import datetime
 5 
from datetime import datetime
6 
import optparse
 6 
import optparse
7 
import smtplib
 7 
import smtplib
8 
from email.mime.text import MIMEText
 8 
from email.mime.text import MIMEText
9 
from email.mime.multipart import MIMEMultipart
 9 
from email.mime.multipart import MIMEMultipart
- 
 
 10 
from BeautifulSoup import BeautifulSoup
- 
 
 11 
from dtr.utils.utils import ungzipResponse
- 
 
 12 
import json
10 
 
 13 
 
11 
con = None
 14 
con = None
12 
parser = optparse.OptionParser()
 15 
parser = optparse.OptionParser()
13 
parser.add_option("-m", "--m", dest="mongoHost",
 16 
parser.add_option("-m", "--m", dest="mongoHost",
14 
                      default="localhost",
 17 
                      default="localhost",
Line 16... Line 19...
16 
                      metavar="mongo_host")
 19 
                      metavar="mongo_host")
17 
 
 20 
 
18 
(options, args) = parser.parse_args()
 21 
(options, args) = parser.parse_args()
19 
 
 22 
 
20 
exceptionList = []
 23 
exceptionList = []
- 
 
 24 
noAttributesSupc = []
- 
 
 25 
 
- 
 
 26 
categoryMap = {3:'Mobiles',5:'Tablets'}
21 
 
 27 
 
22 
headers = {
28 
headers = {
23 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
 29 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
24 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
30 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
25 
            'Accept-Language' : 'en-US,en;q=0.8',
31 
            'Accept-Language' : 'en-US,en;q=0.8',
26 
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
 32 
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
- 
 
 33 
            'Accept-Encoding' : 'gzip,deflate,sdch'
27 
        }
 34 
        }
28 
 
 35 
 
29 
bestSellers = []
 36 
bestSellers = []
30 
now = datetime.now()
 37 
now = datetime.now()
31 
BASE_URL = "snapdeal.com/"
 38 
BASE_URL = "snapdeal.com/"
32 
 
 39 
 
33 
class __RankInfo:
 40 
class __RankInfo:
34
41
35 
    def __init__(self, identifier, rank, category, product_name, page_url):
 42 
    def __init__(self, identifier, rank, category, product_name, page_url, baseColor, ppid, subAttributes, live):
36 
        self.identifier = identifier
 43 
        self.identifier = identifier
37 
        self.rank  = rank
 44 
        self.rank  = rank
38 
        self.category  =category
 45 
        self.category  =category
39 
        self.product_name = product_name
 46 
        self.product_name = product_name
40 
        self.page_url = page_url
47 
        self.page_url = page_url
- 
 
 48 
        self.baseColor = baseColor
- 
 
 49 
        self.ppid = ppid
- 
 
 50 
        self.subAttributes = subAttributes
- 
 
 51 
        self.live = live
- 
 
 52
- 
 
 53 
class __SubAttributes:
- 
 
 54
- 
 
 55 
    def __init__(self, identifier, color, name, value):
- 
 
 56 
        self.identifier = identifier
- 
 
 57 
        self.color = color
- 
 
 58 
        self.name = name
- 
 
 59 
        self.value = value
- 
 
 60 
 
- 
 
 61 
class __Exception:
- 
 
 62
- 
 
 63 
    def __init__(self, identifier, color, desc, pageurl, rank, category, product_name):
- 
 
 64 
        self.identifier = identifier
- 
 
 65 
        self.color = color
- 
 
 66 
        self.desc = desc
- 
 
 67 
        self.pageurl = pageurl
- 
 
 68 
        self.rank = rank
- 
 
 69 
        self.category = category
- 
 
 70 
        self.product_name = product_name
- 
 
 71
- 
 
 72
41 
 
 73 
 
42 
def get_mongo_connection(host=options.mongoHost, port=27017):
 74 
def get_mongo_connection(host=options.mongoHost, port=27017):
43 
    global con
 75 
    global con
44 
    if con is None:
 76 
    if con is None:
45 
        print "Establishing connection %s host and port %d" %(host,port)
 77 
        print "Establishing connection %s host and port %d" %(host,port)
Line 48... Line 80...
48 
        except Exception, e:
 80 
        except Exception, e:
49 
            print e
 81 
            print e
50 
            return None
 82 
            return None
51 
    return con
 83 
    return con
52 
 
 84 
 
- 
 
 85 
def getSoupObject(url):
- 
 
 86 
    print "Getting soup object for"
- 
 
 87 
    print url
- 
 
 88 
    global RETRY_COUNT
- 
 
 89 
    RETRY_COUNT = 1
- 
 
 90 
    while RETRY_COUNT < 10:
- 
 
 91 
        try:
- 
 
 92 
            soup = None
- 
 
 93 
            request = urllib2.Request(url,headers=headers)
- 
 
 94 
            response = urllib2.urlopen(request)
- 
 
 95 
            response_data = ungzipResponse(response)
- 
 
 96 
            response.close()
- 
 
 97 
            try:
- 
 
 98 
                page=response_data.decode("utf-8")
- 
 
 99 
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
- 
 
 100 
            except:
- 
 
 101 
                soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
- 
 
 102 
            if soup is None:
- 
 
 103 
                raise
- 
 
 104 
            return soup
- 
 
 105 
        except Exception as e:
- 
 
 106 
            print e
- 
 
 107 
            print "Retrying"
- 
 
 108 
            RETRY_COUNT = RETRY_COUNT + 1
- 
 
 109 
 
- 
 
 110 
def fetchSupcDetails(p_url, rank, category):
- 
 
 111 
    supcMap = {}
- 
 
 112 
    soup = getSoupObject(p_url)
- 
 
 113 
    attributes =  soup.find('div',{'id':'attributesJson'}).string
- 
 
 114 
    try:
- 
 
 115 
        ppid = soup.find('input',{'id':'pppid'})['value']
- 
 
 116 
    except:
- 
 
 117 
        ppid = p_url[p_url.rfind('/')+1:]
- 
 
 118 
    productName = soup.find('input',{'id':'productNamePDP'})['value']
- 
 
 119 
    supcMap[ppid] = []
- 
 
 120 
    if attributes == "[]":
- 
 
 121 
        supc = soup.find('div',{'id':'defaultSupc'}).string
- 
 
 122 
        r_info = __RankInfo(supc, rank, category, productName, p_url, "", ppid, [], True)
- 
 
 123 
        supcMap[ppid] = [r_info]
- 
 
 124 
    p_info = json.loads(attributes)
- 
 
 125 
    for product in p_info:
- 
 
 126 
        color = product['value']
- 
 
 127 
        supc = product['supc']
- 
 
 128 
        live = product['live']
- 
 
 129 
        r_info = __RankInfo(supc, rank, category, productName, p_url, color, ppid, [], live)
- 
 
 130 
        temp = supcMap.get(ppid)
- 
 
 131 
        temp.append(r_info)
- 
 
 132 
        supcMap[ppid] = temp
- 
 
 133 
        for subAttribute in product['subAttributes']:
- 
 
 134 
            sub_supc = subAttribute['supc']
- 
 
 135 
            sub_value = subAttribute['value']
- 
 
 136 
            sub_name = subAttribute['name']
- 
 
 137 
            subAttr = __SubAttributes(sub_supc, color, sub_name, sub_value)
- 
 
 138 
            subAttributes_list = r_info.subAttributes
- 
 
 139 
            subAttributes_list.append(subAttr)
- 
 
 140 
    return supcMap
- 
 
 141
- 
 
 142
- 
 
 143 
 
53 
def scrapeBestSellerMobiles():
 144 
def scrapeBestSellerMobiles():
54 
    global bestSellers
 145 
    global bestSellers
- 
 
 146 
    bestSellers = []
55 
    rank = 1
 147 
    rank = 1
56 
    for z in [0,20,40,60,80]:
 148 
    for z in [0,20,40,60,80]:
57 
        url = "http://www.snapdeal.com/acors/json/product/get/search/175/%d/20?q=&sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
 149 
        url = "http://www.snapdeal.com/acors/json/product/get/search/175/%d/20?q=&sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
58 
        print url
 - 
 
59 
        request = urllib2.Request(url,headers=headers)
 - 
 
60 
        response = urllib2.urlopen(request)
 - 
 
61 
 
 - 
 
62 
        json_input = response.read()
 150 
        soup = getSoupObject(url)
63 
        info = json.loads(json_input)
 - 
 
64 
        for offer in info['productOfferGroupDtos']:
 151 
        for product in soup.findAll('div',{'class':'outerImg'}):
65 
            #print offer['name']
 152 
            print product.find('a')['pogid']
66 
            #print offer['pageUrl']
 153 
            print product.find('a')['href']
67 
            for identifiers in offer['offers']:
 154 
            supcMap = fetchSupcDetails(product.find('a')['href'],rank, 3)
68 
                r_info = __RankInfo((identifiers['supcs'])[0],rank, None, offer['name'], offer['pageUrl'])
 - 
 
69 
                bestSellers.append(r_info)
 155 
            bestSellers.append(supcMap)
70 
            rank += 1
 156 
            rank = rank + 1
71 
 
 157 
 
72 
def scrapeBestSellerTablets():
 158 
def scrapeBestSellerTablets():
73 
    global bestSellers
 159 
    global bestSellers
74 
    bestSellers = []
 160 
    bestSellers = []
75 
    rank = 1
 161 
    rank = 1
76 
    for z in [0,20,40,60,80]:
 162 
    for z in [0,20,40,60,80]:
77 
        url = "http://www.snapdeal.com/acors/json/product/get/search/133/%d/20?sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
 163 
        url = "http://www.snapdeal.com/acors/json/product/get/search/133/%d/20?sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
78 
        print url
 - 
 
79 
        request = urllib2.Request(url,headers=headers)
 - 
 
80 
        response = urllib2.urlopen(request)
 - 
 
81 
 
 - 
 
82 
        json_input = response.read()
 164 
        soup = getSoupObject(url)
83 
        info = json.loads(json_input)
 165 
        for product in soup.findAll('div',{'class':'outerImg'}):
84 
        for offer in info['productOfferGroupDtos']:
 166 
            print product.find('a')['pogid']
85 
            for identifiers in offer['offers']:
 167 
            print product.find('a')['href']
86 
                r_info = __RankInfo((identifiers['supcs'])[0],rank, None,offer['name'], offer['pageUrl'])
 168 
            supcMap = fetchSupcDetails(product.find('a')['href'],rank, 5)
87 
                bestSellers.append(r_info)
 169 
            bestSellers.append(supcMap)
88 
            rank += 1
 170 
            rank = rank + 1
- 
 
 171 
 
89 
 
 172 
 
90 
def resetRanks(category):
 173 
def resetRanks(category):
91 
    oldRankedItems = get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':3,'category':category})
 - 
 
92 
    for item in oldRankedItems:
 - 
 
93 
        get_mongo_connection().Catalog.MasterData.update({'_id':item['_id']}, {'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
 174 
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':3,'category_id':category},{'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)
94 
 
 175 
 
95 
def commitBestSellers(category):
 176 
def commitBestSellers(category):
96 
    global exceptionList
 177 
    global exceptionList
97 
    print "Rank",
 178 
    for bestSeller in bestSellers:
98 
    print '\t',
 179 
        for mapVal in bestSeller.itervalues():
99 
    print 'Identifier'
 180 
            for v in mapVal:
- 
 
 181 
                col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':v.identifier.strip(),'source_id':3}))
- 
 
 182 
                if len(col) ==0 and len(v.subAttributes) == 0:
- 
 
 183 
                    exObj = __Exception(v.identifier.strip(), v.baseColor, "", v.page_url ,v.rank, category, v.product_name)
100 
    for x in bestSellers:
 184 
                    exceptionList.append(exObj)
101 
        print x.rank,
 185 
                    continue
102 
        print '\t',
 186 
                print v.identifier
103 
        print x.identifier,
 187 
                print v.rank
104 
        col = get_mongo_connection().Catalog.MasterData.find({'identifier':x.identifier.strip()})
 188 
                get_mongo_connection().Catalog.MasterData.update({'identifier':v.identifier,'source_id':3},{'$set' : {'rank':v.rank,'updatedOn':to_java_date(now)}}, multi=True)
105 
        print "count sku",
 189 
                for subAttr in v.subAttributes:
106 
        print '\t',
 190 
                    print "Inside subattr"
107 
        if len(list(col)) == 0:
 191 
                    print vars(subAttr)
- 
 
 192 
                    col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':subAttr.identifier.strip(),'source_id':3}))
108 
            x.category = category
 193 
                    if len(col) ==0:
- 
 
 194 
                        exObj = __Exception(subAttr.identifier.strip(), subAttr.color, subAttr.name+" "+subAttr.value, v.page_url ,v.rank, category, v.product_name)
109 
            exceptionList.append(x)
 195 
                        exceptionList.append(exObj)
110 
        else:
 196 
                    else:
- 
 
 197 
                        print v.identifier
- 
 
 198 
                        print v.rank
111 
            get_mongo_connection().Catalog.MasterData.update({'identifier':x.identifier.strip()}, {'$set' : {'rank':x.rank,'updatedOn':to_java_date(now)}}, multi=True)
 199 
                        get_mongo_connection().Catalog.MasterData.update({'identifier':subAttr.identifier,'source_id':3},{'$set' : {'rank':v.rank,'updatedOn':to_java_date(now)}}, multi=True)
- 
 
 200 
    print exceptionList
112 
 
 201 
 
113 
def sendEmail():
 202 
def sendEmail():
114 
    message="""<html>
 203 
    message="""<html>
115 
            <body>
 204 
            <body>
116 
            <h3>Snapdeal Best Sellers not in master</h3>
 205 
            <h3>Snapdeal Best Sellers not in master</h3>
Line 118... Line 207...
118 
            <thead>
 207 
            <thead>
119 
            <tr><th>Identifier</th>
 208 
            <tr><th>Identifier</th>
120 
            <th>Category</th>
 209 
            <th>Category</th>
121 
            <th>Rank</th>
 210 
            <th>Rank</th>
122 
            <th>Product Name</th>
 211 
            <th>Product Name</th>
- 
 
 212 
            <th>Color</th>
- 
 
 213 
            <th>Description</th>
123 
            <th>Page Url</th>
 214 
            <th>Page Url</th>
124 
            </tr></thead>
 215 
            </tr></thead>
125 
            <tbody>"""
 216 
            <tbody>"""
126 
    for item in exceptionList:
 217 
    for item in exceptionList:
127 
        message+="""<tr>
 218 
        message+="""<tr>
128 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
 219 
        <td style="text-align:center">"""+(item.identifier)+"""</td>
129 
        <td style="text-align:center">"""+(item.category)+"""</td>
 220 
        <td style="text-align:center">"""+(categoryMap.get(item.category))+"""</td>
130 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
 221 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
131 
        <td style="text-align:center">"""+str(item.product_name)+"""</td>
 222 
        <td style="text-align:center">"""+str(item.product_name)+"""</td>
- 
 
 223 
        <td style="text-align:center">"""+item.color+"""</td>
- 
 
 224 
        <td style="text-align:center">"""+item.desc+"""</td>
132 
        <td style="text-align:center">"""+str(BASE_URL+item.page_url)+"""</td>
 225 
        <td style="text-align:center">"""+item.pageurl+"""</td>
133 
        </tr>"""
 226 
        </tr>"""
134 
    message+="""</tbody></table></body></html>"""
 227 
    message+="""</tbody></table></body></html>"""
135 
    print message
 228 
    print message
136 
    #recipients = ['kshitij.sood@saholic.com']
 229 
    #recipients = ['kshitij.sood@saholic.com']
137 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
 230 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
Line 153... Line 246...
153 
        print "Error: unable to send email."
 246 
        print "Error: unable to send email."
154
247
155
248
156 
 
 249 
 
157 
def main():
 250 
def main():
- 
 
 251 
    import time
158 
    scrapeBestSellerMobiles()
 252 
    scrapeBestSellerMobiles()
159 
    if len(bestSellers) > 0:
 253 
    if len(bestSellers) > 0:
160 
        resetRanks('Mobiles')
 254 
        resetRanks(3)
161 
        commitBestSellers('Mobiles')
 255 
        commitBestSellers(3)
162 
    scrapeBestSellerTablets()
 256 
    scrapeBestSellerTablets()
163 
    if len(bestSellers) > 0:
 257 
    if len(bestSellers) > 0:
164 
        resetRanks('Tablets')
 258 
        resetRanks(5)
165 
        commitBestSellers('Tablets')
 259 
        commitBestSellers(5)
166
260
167 
    sendEmail()
 261 
    sendEmail()
168 
 
 262 
 
169 
if __name__=='__main__':
 263 
if __name__=='__main__':
170 
    main()
 264 
    main()
171 

            265