Subversion Repositories SmartDukaan

import urllib2
import simplejson as json
import pymongo
from dtr.utils.utils import to_java_date
from datetime import datetime
import optparse
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from BeautifulSoup import BeautifulSoup
from dtr.utils.utils import ungzipResponse
import json

con = None
parser = optparse.OptionParser()
parser.add_option("-m", "--m", dest="mongoHost",
                      default="localhost",
                      type="string", help="The HOST where the mongo server is running",
                      metavar="mongo_host")

(options, args) = parser.parse_args()

exceptionList = []
noAttributesSupc = []

categoryMap = {3:'Mobiles',5:'Tablets'}

headers = { 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      
            'Accept-Language' : 'en-US,en;q=0.8',                     
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Accept-Encoding' : 'gzip,deflate,sdch'
        }

bestSellers = []
now = datetime.now()
BASE_URL = "snapdeal.com/"

class __RankInfo:
    
    def __init__(self, identifier, rank, category, product_name, page_url, baseColor, ppid, subAttributes, live):
        self.identifier = identifier
        self.rank  = rank
        self.category  =category
        self.product_name = product_name
        self.page_url = page_url 
        self.baseColor = baseColor
        self.ppid = ppid
        self.subAttributes = subAttributes
        self.live = live
        
class __SubAttributes:
    
    def __init__(self, identifier, color, name, value):
        self.identifier = identifier
        self.color = color
        self.name = name
        self.value = value

class __Exception:
    
    def __init__(self, identifier, color, desc, pageurl, rank, category, product_name):
        self.identifier = identifier
        self.color = color
        self.desc = desc
        self.pageurl = pageurl
        self.rank = rank
        self.category = category
        self.product_name = product_name
    
     

def get_mongo_connection(host=options.mongoHost, port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def getSoupObject(url):
    print "Getting soup object for"
    print url
    global RETRY_COUNT
    RETRY_COUNT = 1 
    while RETRY_COUNT < 10:
        try:
            soup = None
            request = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(request)
            response_data = ungzipResponse(response)   
            response.close()
            try:
                page=response_data.decode("utf-8")
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            except:
                soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
            if soup is None:
                raise
            return soup
        except Exception as e:
            print e
            print "Retrying"
            RETRY_COUNT = RETRY_COUNT + 1

def fetchSupcDetails(p_url, rank, category):
    supcMap = {}
    soup = getSoupObject(p_url)
    attributes =  soup.find('div',{'id':'attributesJson'}).string
    try:
        ppid = soup.find('input',{'id':'pppid'})['value']
    except:
        ppid = p_url[p_url.rfind('/')+1:]
    productName = soup.find('input',{'id':'productNamePDP'})['value']
    supcMap[ppid] = []
    if attributes == "[]":
        supc = soup.find('div',{'id':'defaultSupc'}).string
        r_info = __RankInfo(supc, rank, category, productName, p_url, "", ppid, [], True)
        supcMap[ppid] = [r_info]
    p_info = json.loads(attributes)
    for product in p_info:
        color = product['value']
        supc = product['supc']
        live = product['live']
        r_info = __RankInfo(supc, rank, category, productName, p_url, color, ppid, [], live)
        temp = supcMap.get(ppid)
        temp.append(r_info)
        supcMap[ppid] = temp  
        for subAttribute in product['subAttributes']:
            sub_supc = subAttribute['supc']
            sub_value = subAttribute['value']
            sub_name = subAttribute['name']
            subAttr = __SubAttributes(sub_supc, color, sub_name, sub_value)
            subAttributes_list = r_info.subAttributes
            subAttributes_list.append(subAttr)
    return supcMap
            
    

def scrapeBestSellerMobiles():
    global bestSellers
    bestSellers = []
    rank = 1
    for z in [0,20,40,60,80]:
        url = "http://www.snapdeal.com/acors/json/product/get/search/175/%d/20?q=&sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
        soup = getSoupObject(url)
        for product in soup.findAll('div',{'class':'outerImg'}):
            print product.find('a')['pogid']
            print product.find('a')['href']
            supcMap = fetchSupcDetails(product.find('a')['href'],rank, 3)
            bestSellers.append(supcMap)
            rank = rank + 1

def scrapeBestSellerTablets():
    global bestSellers
    bestSellers = []
    rank = 1
    for z in [0,20,40,60,80]:
        url = "http://www.snapdeal.com/acors/json/product/get/search/133/%d/20?sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
        soup = getSoupObject(url)
        for product in soup.findAll('div',{'class':'outerImg'}):
            print product.find('a')['pogid']
            print product.find('a')['href']
            supcMap = fetchSupcDetails(product.find('a')['href'],rank, 5)
            bestSellers.append(supcMap)
            rank = rank + 1


def resetRanks(category):
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':3,'category_id':category},{'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)

def commitBestSellers(category):
    global exceptionList
    for bestSeller in bestSellers:
        for mapVal in bestSeller.itervalues():
            for v in mapVal:
                col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':v.identifier.strip(),'source_id':3}))
                if len(col) ==0 and len(v.subAttributes) == 0:
                    exObj = __Exception(v.identifier.strip(), v.baseColor, "", v.page_url ,v.rank, category, v.product_name)
                    exceptionList.append(exObj)
                    continue
                print v.identifier
                print v.rank
                get_mongo_connection().Catalog.MasterData.update({'identifier':v.identifier,'source_id':3},{'$set' : {'rank':v.rank,'updatedOn':to_java_date(now)}}, multi=True)
                for subAttr in v.subAttributes:
                    print "Inside subattr"
                    print vars(subAttr)
                    col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':subAttr.identifier.strip(),'source_id':3}))
                    if len(col) ==0:
                        exObj = __Exception(subAttr.identifier.strip(), subAttr.color, subAttr.name+" "+subAttr.value, v.page_url ,v.rank, category, v.product_name)
                        exceptionList.append(exObj)
                    else:
                        print v.identifier
                        print v.rank
                        get_mongo_connection().Catalog.MasterData.update({'identifier':subAttr.identifier,'source_id':3},{'$set' : {'rank':v.rank,'updatedOn':to_java_date(now)}}, multi=True)
    print exceptionList

def sendEmail():
    message="""<html>
            <body>
            <h3>Snapdeal Best Sellers not in master</h3>
            <table border="1" style="width:100%;">
            <thead>
            <tr><th>Identifier</th>
            <th>Category</th>
            <th>Rank</th>
            <th>Product Name</th>
            <th>Color</th>
            <th>Description</th>
            <th>Page Url</th>
            </tr></thead>
            <tbody>"""
    for item in exceptionList:
        message+="""<tr>
        <td style="text-align:center">"""+(item.identifier)+"""</td>
        <td style="text-align:center">"""+(categoryMap.get(item.category))+"""</td>
        <td style="text-align:center">"""+str(item.rank)+"""</td>
        <td style="text-align:center">"""+str(item.product_name)+"""</td>
        <td style="text-align:center">"""+item.color+"""</td>
        <td style="text-align:center">"""+item.desc+"""</td>
        <td style="text-align:center">"""+item.pageurl+"""</td>
        </tr>"""
    message+="""</tbody></table></body></html>"""
    print message
    #recipients = ['kshitij.sood@saholic.com']
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
    msg = MIMEMultipart()
    msg['Subject'] = "Snapdeal Best Sellers" + ' - ' + str(datetime.now())
    msg['From'] = ""
    msg['To'] = ",".join(recipients)
    msg.preamble = "Snapdeal Best Sellers" + ' - ' + str(datetime.now())
    html_msg = MIMEText(message, 'html')
    msg.attach(html_msg)
    
    smtpServer = smtplib.SMTP('localhost')
    smtpServer.set_debuglevel(1)
    sender = 'dtr@shop2020.in'
    try:
        smtpServer.sendmail(sender, recipients, msg.as_string())
        print "Successfully sent email"
    except:
        print "Error: unable to send email."
            
            

def main():
    import time
    scrapeBestSellerMobiles()
    if len(bestSellers) > 0:
        resetRanks(3)
        commitBestSellers(3)
    scrapeBestSellerTablets()
    if len(bestSellers) > 0:
        resetRanks(5)
        commitBestSellers(5)
        
    sendEmail()

if __name__=='__main__':
    main()