Subversion Repositories SmartDukaan

Rev

Rev 15869 | Rev 16019 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 15869 Rev 15895
Line 6... Line 6...
6 
import optparse
 6 
import optparse
7 
from datetime import datetime
 7 
from datetime import datetime
8 
import smtplib
 8 
import smtplib
9 
from email.mime.text import MIMEText
 9 
from email.mime.text import MIMEText
10 
from email.mime.multipart import MIMEMultipart
 10 
from email.mime.multipart import MIMEMultipart
- 
 
 11 
from dtr.utils import ShopCluesScraper
- 
 
 12 
import traceback
11 
 
 13 
 
12 
con = None
 14 
con = None
13 
parser = optparse.OptionParser()
 15 
parser = optparse.OptionParser()
14 
parser.add_option("-m", "--m", dest="mongoHost",
 16 
parser.add_option("-m", "--m", dest="mongoHost",
15 
                      default="localhost",
 17 
                      default="localhost",
Line 20... Line 22...
20 
 
 22 
 
21 
 
 23 
 
22 
exceptionList = []
 24 
exceptionList = []
23 
bestSellers = []
 25 
bestSellers = []
24 
baseUrl = "http://m.shopclues.com/products/getProductList/mobiles:top-selling-mobiles-and-tablets.html/%s/page=%s"
 26 
baseUrl = "http://m.shopclues.com/products/getProductList/mobiles:top-selling-mobiles-and-tablets.html/%s/page=%s"
- 
 
 27 
headers = {
- 
 
 28 
            'User-Agent':'Mozilla/5.0 (Linux; Android 4.3; Nexus 7 Build/JSS15Q) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.72 Safari/537.36',
- 
 
 29 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 
 
 30 
            'Accept-Language' : 'en-US,en;q=0.8',
- 
 
 31 
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
- 
 
 32 
            'Connection':'keep-alive',
- 
 
 33 
            'Accept-Encoding' : 'gzip,deflate,sdch'
- 
 
 34 
        }
25 
now = datetime.now()
 35 
now = datetime.now()
- 
 
 36 
sc = ShopCluesScraper.ShopCluesScraper()
26 
 
 37 
 
27 
 
 38 
 
28 
class __ProductInfo:
 39 
class __ProductInfo:
29
40
30 
    def __init__(self, identifier, rank, url, available_price, in_stock, codAvailable, source_product_name, thumbnail, coupon):
 41 
    def __init__(self, identifier, rank, url, available_price, in_stock, codAvailable, source_product_name, thumbnail, coupon):
Line 55... Line 66...
55 
    global RETRY_COUNT
 66 
    global RETRY_COUNT
56 
    RETRY_COUNT = 1
67 
    RETRY_COUNT = 1
57 
    while RETRY_COUNT < 10:
 68 
    while RETRY_COUNT < 10:
58 
        try:
 69 
        try:
59 
            soup = None
 70 
            soup = None
60 
            request = urllib2.Request(url)
 71 
            request = urllib2.Request(url, headers)
61 
            request.add_header('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
 - 
 
62 
            request.add_header('Accept-Language','en-US,en;q=0.8,hi;q=0.6')
 - 
 
63 
            request.add_header('Connection','keep-alive')
 - 
 
64 
            request.add_header('User-Agent','Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36')
 - 
 
65
- 
 
66 
            response = urllib2.urlopen(request)
72 
            response = urllib2.urlopen(request)
67 
            response_data = response.read()
 73 
            response_data = response.read()
68 
            response.close()
 74 
            response.close()
69 
            try:
 75 
            try:
70 
                page=response_data.decode("utf-8")
 76 
                page=response_data.decode("utf-8")
Line 96... Line 102...
96 
            info_tag =  productDiv.find('a')
 102 
            info_tag =  productDiv.find('a')
97 
            link = info_tag['href']
 103 
            link = info_tag['href']
98 
            scin = info_tag['data-id'].strip()
 104 
            scin = info_tag['data-id'].strip()
99 
            print link
 105 
            print link
100 
            print scin
 106 
            print scin
- 
 
 107 
            productName = productDiv.find('div',{'class':'pdt-name'}).string
- 
 
 108 
            try:
- 
 
 109 
                productInfo = sc.read(link)
- 
 
 110 
            except Exception as e:
- 
 
 111 
                traceback.print_exc()
- 
 
 112 
                continue
101 
            product = list(get_mongo_connection().Catalog.MasterData.find({'source_id':5,'identifier':scin}))
 113 
            product = list(get_mongo_connection().Catalog.MasterData.find({'source_id':5,'identifier':scin}))
102 
            if len(product) > 0:
 114 
            if len(product) > 0:
103 
                get_mongo_connection().Catalog.MasterData.update({'source_id':5,'identifier':scin},{"$set":{'rank':rank}})
 115 
                get_mongo_connection().Catalog.MasterData.update({'source_id':5,'identifier':scin},{"$set":{'rank':rank, 'available_price':productInfo['price'], \
- 
 
 116 
                                                                                                            'in_stock':productInfo['inStock'], 'codAvailable':productInfo['isCod'], \
- 
 
 117 
                                                                                                            'coupon':productInfo['coupon']}})
104 
            else:
 118 
            else:
105 
                #Lets bundle product by finding similar url pattern
 119 
                #Lets bundle product by finding similar url pattern
106 
                uri = link.replace('http://m.shopclues.com','')
 120 
                uri = link.replace('http://m.shopclues.com','').replace(".html","")
- 
 
 121 
                try:
- 
 
 122 
                    int(uri[uri.rfind('-')+1:])
- 
 
 123 
                    uri =  uri[:uri.rfind('-')]
- 
 
 124 
                except:
- 
 
 125 
                    pass
107 
                product = list(get_mongo_connection().Catalog.MasterData.find({'marketPlaceUrl':{'$regex': uri}}))
 126 
                product = list(get_mongo_connection().Catalog.MasterData.find({'source_id':5,'marketPlaceUrl':{'$regex': uri}}))
- 
 
 127 
                toBundle = __ProductInfo(scin, rank, link, productInfo['price'], productInfo['inStock'],productInfo['isCod'], productName, "" ,productInfo['coupon'])
108 
                if len(product) > 0:
 128 
                if len(product) > 0:
109 
                    bundleNewProduct(product[0],)
 129 
                    bundleNewProduct(product[0], toBundle)
- 
 
 130 
                else:
- 
 
 131 
                    exceptionList.append(toBundle)
110 
        page = page+1
 132 
        page = page+1
111
133
112 
def bundleNewProduct(existingProduct, toBundle):
 134 
def bundleNewProduct(existingProduct, toBundle):
- 
 
 135 
    print "Adding new product"
- 
 
 136 
    try:
- 
 
 137 
        max_id = list(get_mongo_connection().Catalog.MasterData.find().sort([('_id',pymongo.DESCENDING)]).limit(1))
- 
 
 138 
        existingProduct['_id'] = max_id[0]['_id'] + 1
- 
 
 139 
        existingProduct['addedOn'] = to_java_date(datetime.now())
- 
 
 140 
        existingProduct['available_price'] = toBundle.available_price
- 
 
 141 
        existingProduct['updatedOn'] = to_java_date(datetime.now())
- 
 
 142 
        existingProduct['codAvailable'] = toBundle.codAvailable
- 
 
 143 
        existingProduct['coupon'] = toBundle.coupon
- 
 
 144 
        existingProduct['identifier'] = toBundle.identifier
- 
 
 145 
        existingProduct['in_stock'] = toBundle.in_stock
- 
 
 146 
        existingProduct['marketPlaceUrl'] = toBundle.url
- 
 
 147 
        existingProduct['rank'] = toBundle.rank
- 
 
 148 
        existingProduct['source_product_name'] = toBundle.source_product_name
- 
 
 149 
        existingProduct['url'] = toBundle.url
- 
 
 150 
        get_mongo_connection().Catalog.MasterData.insert(existingProduct)
- 
 
 151 
        return {1:'Data added successfully.'}
- 
 
 152 
    except Exception as e:
- 
 
 153 
        print e
- 
 
 154 
        return {0:'Unable to add data.'}
- 
 
 155 
 
- 
 
 156 
def exceptionItems():
- 
 
 157 
    for item in exceptionList:
- 
 
 158 
        print vars(item)
113 
    pass
159
114 
 
 160 
 
115 
def main():
 161 
def main():
116 
    scrapeBestSellers()
 162 
    scrapeBestSellers()
- 
 
 163 
    exceptionItems()
117
164
118 
if __name__=='__main__':
 165 
if __name__=='__main__':
119 
    main()
 166 
    main()
120 

            167