| 158 |
ashish |
1 |
'''
|
|
|
2 |
Created on 16-May-2010
|
|
|
3 |
|
|
|
4 |
@author: gaurav
|
|
|
5 |
'''
|
|
|
6 |
|
|
|
7 |
|
|
|
8 |
from scrapy.spider import BaseSpider
|
|
|
9 |
from scrapy.selector import HtmlXPathSelector
|
|
|
10 |
from scrapy.http import Request
|
|
|
11 |
|
|
|
12 |
from demo.items import DemoItem
|
|
|
13 |
from scrapy.contrib.spidermiddleware import referer
|
|
|
14 |
from scrapy.http.headers import Headers
|
|
|
15 |
from scrapy.http.request.form import FormRequest
|
|
|
16 |
from scrapy.log import msg
|
|
|
17 |
from scrapy.http.response import Response
|
|
|
18 |
|
| 226 |
ashish |
19 |
from html2text.unescaping import *
|
|
|
20 |
from datastore.DataAccessor import *
|
|
|
21 |
from datastore.DataCodeAccessor import *
|
| 158 |
ashish |
22 |
|
|
|
23 |
|
| 226 |
ashish |
24 |
|
| 158 |
ashish |
25 |
class indiaplaza_spider(BaseSpider):
|
|
|
26 |
|
|
|
27 |
def __init__(self):
|
| 226 |
ashish |
28 |
initialize_table()
|
|
|
29 |
da = DataHelper()
|
|
|
30 |
#INDIAPLAZA_DOMAINNAME = "indiaplaza"
|
|
|
31 |
INDIAPLAZA_DOMAINNAME = get_code_word("INDIAPLAZA_DOMAINNAME")
|
|
|
32 |
self.domain_name = INDIAPLAZA_DOMAINNAME
|
|
|
33 |
#INDIAPLAZA_CT = 18
|
|
|
34 |
#INDIAPLAZA_CT = int(get_code_word("INDIAPLAZA_CT"))
|
|
|
35 |
#INDIAPLAZA_NO = 1
|
|
|
36 |
CT = int(da.get_extra_vars('indiaplaza_count'))
|
|
|
37 |
NO = 1
|
|
|
38 |
if CT>18:
|
|
|
39 |
NO = CT
|
|
|
40 |
#INDIAPLAZA_URL = "http://www.indiaplaza.in/mobile-phones-Mobiles-1.htm?PageNo="
|
|
|
41 |
INDIAPLAZA_URL = get_code_word("INDIAPLAZA_URL")
|
|
|
42 |
while(NO<=CT):
|
|
|
43 |
url1 = INDIAPLAZA_URL + str(NO)
|
|
|
44 |
self.start_urls.append(url1)
|
|
|
45 |
NO=NO+1
|
|
|
46 |
|
| 158 |
ashish |
47 |
def start_requests(self):
|
| 226 |
ashish |
48 |
#adding entry for the supplier i.e its name and site
|
|
|
49 |
#INDIAPLAZA_HOMEPAGE = "www.indiaplaza.com"
|
|
|
50 |
INDIAPLAZA_HOMEPAGE = get_code_word("INDIAPLAZA_HOMEPAGE")
|
| 178 |
ashish |
51 |
da = DataHelper()
|
| 226 |
ashish |
52 |
da.add_supplier(self.domain_name, INDIAPLAZA_HOMEPAGE)
|
| 158 |
ashish |
53 |
listreq = []
|
| 226 |
ashish |
54 |
|
|
|
55 |
#for each request a referer has to be set
|
|
|
56 |
#INDIAPLAZA_REFERER = "www.google.com/search"
|
|
|
57 |
INDIAPLAZA_REFERER = get_code_word("INDIAPLAZA_REFERER")
|
| 158 |
ashish |
58 |
for url1 in self.start_urls:
|
| 226 |
ashish |
59 |
request = Request(url = str(url1), callback=self.parse, dont_filter=True)
|
|
|
60 |
request.headers.setdefault("Referer", INDIAPLAZA_REFERER)
|
| 158 |
ashish |
61 |
listreq.append(request)
|
|
|
62 |
return listreq
|
| 226 |
ashish |
63 |
|
| 158 |
ashish |
64 |
def parse(self, response):
|
| 226 |
ashish |
65 |
da = DataHelper()
|
|
|
66 |
#INDIAPLAZA_URL1 = "http://www.indiaplaza.in"
|
|
|
67 |
INDIAPLAZA_URL1 = get_code_word("INDIAPLAZA_URL1")
|
| 158 |
ashish |
68 |
hxs = HtmlXPathSelector(response)
|
| 226 |
ashish |
69 |
#INDIAPLAZA_XPATH1 = '//tr/td/table[@id="browsesku"]'
|
|
|
70 |
INDIAPLAZA_XPATH1 = get_code_word("INDIAPLAZA_XPATH1")
|
|
|
71 |
phone_info = hxs.select(INDIAPLAZA_XPATH1)
|
| 158 |
ashish |
72 |
items = []
|
| 226 |
ashish |
73 |
#INDIAPLAZA_XPATH2 = './/div[@class="skuimg"]/a/@title'
|
|
|
74 |
INDIAPLAZA_XPATH2 = get_code_word("INDIAPLAZA_XPATH2")
|
|
|
75 |
#INDIAPLAZA_XPATH3 = './/div[@class="skuimg"]/a/@href'
|
|
|
76 |
INDIAPLAZA_XPATH3 = get_code_word("INDIAPLAZA_XPATH3")
|
|
|
77 |
if not phone_info:
|
|
|
78 |
ct = int(da.get_extra_vars('indiaplaza_count'))
|
|
|
79 |
if ct>18:
|
|
|
80 |
fails = int(da.get_extra_vars('indiaplaza_fails'))
|
|
|
81 |
fails = fails+1
|
|
|
82 |
da.set_extra_vars('indiaplaza_fails',str(fails),'')
|
|
|
83 |
if fails > 0:
|
|
|
84 |
da.set_extra_vars('indiaplaza_flag','FALSE','')
|
|
|
85 |
da.set_extra_vars('indiaplaza_fails',str(fails),'')
|
|
|
86 |
else:
|
|
|
87 |
for i in phone_info:
|
|
|
88 |
item = {}
|
|
|
89 |
item['title'] = i.select(INDIAPLAZA_XPATH2)[0].extract()
|
|
|
90 |
item['url'] = i.select(INDIAPLAZA_XPATH3)[0].extract()
|
|
|
91 |
items.append(item)
|
|
|
92 |
|
|
|
93 |
for item in items:
|
|
|
94 |
str2 = INDIAPLAZA_URL1 + str(item['url'])
|
|
|
95 |
da.add_ipbasic(item['title'],unescape(str2))
|
|
|
96 |
|
| 158 |
ashish |
97 |
SPIDER = indiaplaza_spider()
|