Subversion Repositories SmartDukaan

Rev

Rev 183 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 183 Rev 231
Line 15... Line 15...
15 
from scrapy.log import msg
 15 
from scrapy.log import msg
16 
from scrapy.http.response import Response
 16 
from scrapy.http.response import Response
17 
 
 17 
 
18 
from datastore import DataAccessor
 18 
from datastore import DataAccessor
19 
from datastore.DataAccessor import DataHelper
 19 
from datastore.DataAccessor import DataHelper
20 
 
 - 
 
- 
 
 20 
from html2text.unescaping import *
21 
 
 21 
 
22 
class mobilestore_spider(BaseSpider):
 22 
class mobilestore_spider(BaseSpider):
23
23
24 
    def __init__(self):
24 
    def __init__(self):
- 
 
 25 
       MOBILESTORE_DOMAINNAME = "mobilestore"
25 
       self.domain_name = "mobilestore"
 26 
       self.domain_name = MOBILESTORE_DOMAINNAME
26 
       str1 = "http://www.themobilestore.in/sitemap.xml"
 27 
       MOBILESTORE_URL = "http://www.themobilestore.in/sitemap.xml"
27 
       self.start_urls.append(str1)
 28 
       self.start_urls.append(MOBILESTORE_URL)
28
- 
 
29
29
30 
    def start_requests(self):
 30 
    def start_requests(self):
- 
 
 31 
        #adding entry for the supplier i.e its name and site
- 
 
 32 
        MOBILESTORE_HOMEPAGE = "www.themobilestore.in"
31 
        da = DataHelper()
 33 
        da = DataHelper()
32 
        da.add_supplier(self.domain_name, "www.themobilestore.in")
 34 
        da.add_supplier(self.domain_name, MOBILESTORE_HOMEPAGE)
33 
        listreq = []
 35 
        listreq = []
- 
 
 36
- 
 
 37 
        #for each request a referer has to be set
- 
 
 38 
        MOBILESTORE_REFERER = "www.google.com/search"
34 
        for url1 in self.start_urls:
 39 
        for url1 in self.start_urls:
35 
            request = Request(url = str(url1), callback=self.parse)
 40 
            request = Request(url = str(url1), callback=self.parse)
36 
            request.headers.setdefault("Referer", "http://www.mobilestore.in")
 41 
            request.headers.setdefault("Referer", MOBILESTORE_REFERER)
37 
            listreq.append(request)
 42 
            listreq.append(request)
38 
        return listreq
 43 
        return listreq
39
44
40 
    def parse(self, response):
 45 
    def parse(self, response):
41 
        da = DataHelper()
 46 
        da = DataHelper()
42 
        hxs = HtmlXPathSelector(response)
 47 
        hxs = HtmlXPathSelector(response)
- 
 
 48 
        MOBILESTORE_XPATH1 = '//url/loc/text()'
43 
        phone_urls = hxs.select('//url/loc/text()')
 49 
        phone_urls = hxs.select(MOBILESTORE_XPATH1)
- 
 
 50 
         #elements in chk_list are specific to this site for determining valid sites
44 
        ct = 1
 51 
        MOBILESTORE_CHKLIST1 = ["Mobile_Phones"]
45 
        for i in phone_urls:
 52 
        for i in phone_urls:
46 
            site = i.extract()
 53 
            site = i.extract()
- 
 
 54 
            site = unescape(site)
47 
            pos1 = pos2 = pos3 = 0
 55 
            pos1 = pos2 = pos3 = 0
48 
            temp =""
 56 
            temp =""
49 
            pos1 = site.rfind('/')
 57 
            pos1 = site.rfind('/')
50 
            if pos1 != -1:
 58 
            if pos1 != -1:
51 
                pos2 = site.rfind('/',0,pos1-1)
 59 
                pos2 = site.rfind('/',0,pos1-1)
52 
            if pos2 != -1:
60 
            if pos2 != -1:
53 
                pos3 = site.rfind('/',0,pos2-1)
 61 
                pos3 = site.rfind('/',0,pos2-1)
54 
            if pos3 > 0:
 62 
            if pos3 > 0:
55 
                temp = site[pos3+1:pos2]
 63 
                temp = site[pos3+1:pos2]
56 
            if temp == "Mobile_Phones":
 - 
 
57 
                #print str(ct) + " " + site
 64 
            # adding valid urls to the DB from the site-map
58 
                #print "\n"
 65 
            for m in MOBILESTORE_CHKLIST1:
59 
                #ct = ct +1
66 
                if temp == m:
60 
                da.add_mobstoreurl(site)
67 
                    da.add_mobstoreurl(site)
61 
SPIDER = mobilestore_spider()
 68 
SPIDER = mobilestore_spider()
62 

            69