Subversion Repositories SmartDukaan

import urllib2
import simplejson as json
import pymongo
from dtr.utils.utils import to_java_date
from datetime import datetime
import optparse
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from BeautifulSoup import BeautifulSoup
from dtr.utils.utils import ungzipResponse
import json
import chardet
from shop2020.utils.EmailAttachmentSender import get_attachment_part
from shop2020.utils import EmailAttachmentSender

con = None
parser = optparse.OptionParser()
parser.add_option("-m", "--m", dest="mongoHost",
                      default="localhost",
                      type="string", help="The HOST where the mongo server is running",
                      metavar="mongo_host")

(options, args) = parser.parse_args()

exceptionList = []
noAttributesSupc = []

categoryMap = {3:'Mobiles',5:'Tablets'}

headers = { 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      
            'Accept-Language' : 'en-US,en;q=0.8',                     
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Accept-Encoding' : 'gzip,deflate,sdch'
        }

bestSellers = []
now = datetime.now()
BASE_URL = "snapdeal.com/"

class __RankInfo:
    
    def __init__(self, identifier, rank, category, product_name, page_url, baseColor, ppid, subAttributes, live):
        self.identifier = identifier
        self.rank  = rank
        self.category  =category
        self.product_name = product_name
        self.page_url = page_url 
        self.baseColor = baseColor
        self.ppid = ppid
        self.subAttributes = subAttributes
        self.live = live
        
class __SubAttributes:
    
    def __init__(self, identifier, color, name, value):
        self.identifier = identifier
        self.color = color
        self.name = name
        self.value = value

class __Exception:
    
    def __init__(self, identifier, color, desc, pageurl, rank, category, product_name):
        self.identifier = identifier
        self.color = color
        self.desc = desc
        self.pageurl = pageurl
        self.rank = rank
        self.category = category
        self.product_name = product_name
    
     

def get_mongo_connection(host=options.mongoHost, port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

def getSoupObject(url):
    print "Getting soup object for"
    print url
    global RETRY_COUNT
    RETRY_COUNT = 1 
    while RETRY_COUNT < 10:
        try:
            soup = None
            request = urllib2.Request(url,headers=headers)
            response = urllib2.urlopen(request)
            response_data = ungzipResponse(response)   
            response.close()
            try:
                page=response_data.decode("utf-8")
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            except:
                soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
            if soup is None:
                raise
            return soup
        except Exception as e:
            print e
            print "Retrying"
            RETRY_COUNT = RETRY_COUNT + 1

def fetchSupcDetails(p_url, rank, category):
    supcMap = {}
    soup = getSoupObject(p_url)
    attributes =  soup.find('div',{'id':'attributesJson'}).string
    try:
        ppid = soup.find('input',{'id':'pppid'})['value']
    except:
        ppid = p_url[p_url.rfind('/')+1:]
    productName = soup.find('input',{'id':'productNamePDP'})['value']
    supcMap[ppid] = []
    if attributes == "[]":
        supc = soup.find('div',{'id':'defaultSupc'}).string
        r_info = __RankInfo(supc, rank, category, productName, p_url, "", ppid, [], True)
        supcMap[ppid] = [r_info]
    p_info = json.loads(attributes)
    for product in p_info:
        color = product['value']
        supc = product['supc']
        live = product['live']
        r_info = __RankInfo(supc, rank, category, productName, p_url, color, ppid, [], live)
        temp = supcMap.get(ppid)
        temp.append(r_info)
        supcMap[ppid] = temp  
        if product['subAttributes'] is not None:
            for subAttribute in product['subAttributes']:
                sub_supc = subAttribute['supc']
                sub_value = subAttribute['value']
                sub_name = subAttribute['name']
                subAttr = __SubAttributes(sub_supc, color, sub_name, sub_value)
                subAttributes_list = r_info.subAttributes
                subAttributes_list.append(subAttr)
    return supcMap
            
    

def scrapeBestSellerMobiles():
    global bestSellers
    bestSellers = []
    rank = 1
    for z in [0,20,40,60,80]:
        url = "http://www.snapdeal.com/acors/json/product/get/search/175/%d/20?q=&sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
        soup = getSoupObject(url)
        for product in soup.findAll('a',{'class':'dp-widget-link'}):
            print product['href']
            try:
                supcMap = fetchSupcDetails(product['href'],rank, 3)
            except:
                continue
            print "supcMap ",supcMap
            bestSellers.append(supcMap)
            rank = rank + 1

def scrapeBestSellerTablets():
    global bestSellers
    bestSellers = []
    rank = 1
    for z in [0,20,40,60,80]:
        url = "http://www.snapdeal.com/acors/json/product/get/search/133/%d/20?sort=bstslr&keyword=&clickSrc=&viewType=List&lang=en&snr=false" %(z)
        soup = getSoupObject(url)
        for product in soup.findAll('a',{'class':'dp-widget-link'}):
            print product['href']
            try:
                supcMap = fetchSupcDetails(product['href'],rank, 5)
            except:
                continue
            print "supcMap ",supcMap
            bestSellers.append(supcMap)
            rank = rank + 1


def resetRanks(category):
    get_mongo_connection().Catalog.MasterData.update({'rank':{'$gt':0},'source_id':3,'category_id':category},{'$set' : {'rank':0,'updatedOn':to_java_date(now)}}, multi=True)

def commitBestSellers(category):
    global exceptionList
    for bestSeller in bestSellers:
        for mapVal in bestSeller.itervalues():
            for v in mapVal:
                col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':v.identifier.strip(),'source_id':3}))
                if len(col) ==0 and len(v.subAttributes) == 0:
                    exObj = __Exception(v.identifier.strip(), v.baseColor, "", v.page_url ,v.rank, category, v.product_name)
                    exceptionList.append(exObj)
                    continue
                print v.identifier
                print v.rank
                get_mongo_connection().Catalog.MasterData.update({'identifier':v.identifier,'source_id':3},{'$set' : {'rank':v.rank,'updatedOn':to_java_date(now)}}, multi=True)
                for subAttr in v.subAttributes:
                    print "Inside subattr"
                    print vars(subAttr)
                    col = list(get_mongo_connection().Catalog.MasterData.find({'identifier':subAttr.identifier.strip(),'source_id':3}))
                    if len(col) ==0:
                        exObj = __Exception(subAttr.identifier.strip(), subAttr.color, subAttr.name+" "+subAttr.value, v.page_url ,v.rank, category, v.product_name)
                        exceptionList.append(exObj)
                    else:
                        print v.identifier
                        print v.rank
                        get_mongo_connection().Catalog.MasterData.update({'identifier':subAttr.identifier,'source_id':3},{'$set' : {'rank':v.rank,'updatedOn':to_java_date(now)}}, multi=True)
    print exceptionList

def sendEmail():
    message="""<html>
            <body>
            <h3>Snapdeal Best Sellers not in master</h3>
            <table border="1" style="width:100%;">
            <thead>
            <tr><th>Identifier</th>
            <th>Category</th>
            <th>Rank</th>
            <th>Product Name</th>
            <th>Color</th>
            <th>Description</th>
            <th>Page Url</th>
            </tr></thead>
            <tbody>"""
    for item in exceptionList:
        encoding =  chardet.detect(item.pageurl)
        try:
            message+="""<tr>
            <td style="text-align:center">"""+(item.identifier)+"""</td>
            <td style="text-align:center">"""+(categoryMap.get(item.category))+"""</td>
            <td style="text-align:center">"""+str(item.rank)+"""</td>
            <td style="text-align:center">"""+str(item.product_name)+"""</td>
            <td style="text-align:center">"""+item.color+"""</td>
            <td style="text-align:center">"""+item.desc+"""</td>
            <td style="text-align:center">"""+item.pageurl+"""</td>
            </tr>"""
        except:
            pass
    message+="""</tbody></table></body></html>"""
    print message
    #recipients = ['kshitij.sood@saholic.com']
    recipients = ['kshitij.sood@saholic.com','ritesh.chauhan@saholic.com','aishwarya.singh@saholic.com']
    EmailAttachmentSender.mail_send_grid("dtr@smartdukaan.com","apikey", "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", recipients, "Snapdeal Best Sellers",message ,[],[],[])              
            

def main():
    import time
    scrapeBestSellerMobiles()
    if len(bestSellers) > 0:
        resetRanks(3)
        commitBestSellers(3)
    scrapeBestSellerTablets()
    if len(bestSellers) > 0:
        resetRanks(5)
        commitBestSellers(5)
    print bestSellers
    sendEmail()

if __name__=='__main__':
    main()