| Line 8... |
Line 8... |
| 8 |
from email.mime.text import MIMEText
|
8 |
from email.mime.text import MIMEText
|
| 9 |
from email.mime.multipart import MIMEMultipart
|
9 |
from email.mime.multipart import MIMEMultipart
|
| 10 |
from dtr.utils.utils import fetchResponseUsingProxy, get_mongo_connection, ungzipResponse
|
10 |
from dtr.utils.utils import fetchResponseUsingProxy, get_mongo_connection, ungzipResponse
|
| 11 |
import json
|
11 |
import json
|
| 12 |
import urllib
|
12 |
import urllib
|
| - |
|
13 |
import chardet
|
| 13 |
|
14 |
|
| 14 |
|
15 |
|
| 15 |
con = None
|
16 |
con = None
|
| 16 |
parser = optparse.OptionParser()
|
17 |
parser = optparse.OptionParser()
|
| 17 |
parser.add_option("-m", "--m", dest="mongoHost",
|
18 |
parser.add_option("-m", "--m", dest="mongoHost",
|
| Line 85... |
Line 86... |
| 85 |
rank = rank +1
|
86 |
rank = rank +1
|
| 86 |
if rank >100:
|
87 |
if rank >100:
|
| 87 |
break
|
88 |
break
|
| 88 |
titleTag = tag.find('p', {'class' : 'product_title'})
|
89 |
titleTag = tag.find('p', {'class' : 'product_title'})
|
| 89 |
source_product_name = titleTag.text
|
90 |
source_product_name = titleTag.text
|
| - |
|
91 |
encoding = chardet.detect(source_product_name)
|
| - |
|
92 |
try:
|
| - |
|
93 |
source_product_name = source_product_name.decode(encoding.get('encoding'))
|
| - |
|
94 |
except:
|
| - |
|
95 |
source_product_name = source_product_name.decode('latin-1')
|
| 90 |
productUrl = titleTag.find('a').get('href')
|
96 |
productUrl = titleTag.find('a').get('href')
|
| 91 |
productUrl = 'http://www.homeshop18.com'+str(productUrl)
|
97 |
productUrl = 'http://www.homeshop18.com'+str(productUrl)
|
| 92 |
inStock = 1
|
98 |
inStock = 1
|
| 93 |
identifier = tag['id'].split('_')[1]
|
99 |
identifier = tag['id'].split('_')[1]
|
| 94 |
available_price = long(tag.find('p',{'class':'price clearfix'}).find('b').text.split(' ')[1])
|
100 |
available_price = long(tag.find('p',{'class':'price clearfix'}).find('b').text.split(' ')[1])
|
| Line 113... |
Line 119... |
| 113 |
rank = rank +1
|
119 |
rank = rank +1
|
| 114 |
if rank >100:
|
120 |
if rank >100:
|
| 115 |
break
|
121 |
break
|
| 116 |
titleTag = tag.find('p', {'class' : 'product_title'})
|
122 |
titleTag = tag.find('p', {'class' : 'product_title'})
|
| 117 |
source_product_name = titleTag.text
|
123 |
source_product_name = titleTag.text
|
| - |
|
124 |
encoding = chardet.detect(source_product_name)
|
| - |
|
125 |
try:
|
| - |
|
126 |
source_product_name = source_product_name.decode(encoding.get('encoding'))
|
| - |
|
127 |
except:
|
| - |
|
128 |
source_product_name = source_product_name.decode('latin-1')
|
| 118 |
productUrl = titleTag.find('a').get('href')
|
129 |
productUrl = titleTag.find('a').get('href')
|
| 119 |
productUrl = 'http://www.homeshop18.com'+str(productUrl)
|
130 |
productUrl = 'http://www.homeshop18.com'+str(productUrl)
|
| 120 |
inStock = 1
|
131 |
inStock = 1
|
| 121 |
identifier = tag['id'].split('_')[1]
|
132 |
identifier = tag['id'].split('_')[1]
|
| 122 |
available_price = long(tag.find('p',{'class':'price clearfix'}).find('b').text.split(' ')[1])
|
133 |
available_price = long(tag.find('p',{'class':'price clearfix'}).find('b').text.split(' ')[1])
|