Subversion Repositories SmartDukaan

Rev

Rev 228 | Rev 270 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 228 Rev 259
Line 19... Line 19...
19 
from datastore.DataCodeAccessor import *
 19 
from datastore.DataCodeAccessor import *
20 
from html2text.unescaping import *
 20 
from html2text.unescaping import *
21 
from elixir import *
 21 
from elixir import *
22 
 
 22 
 
23 
class infi_spider(BaseSpider):
 23 
class infi_spider(BaseSpider):
- 
 
 24 
    """
- 
 
 25 
    Documentation for class infi_spider
- 
 
 26 
    This spider collects the information for the individual phones
- 
 
 27 
    and store them in table datastore_datadefinition_infibeam_data
24
28 
    """
25 
    def __init__(self):
29 
    def __init__(self):
- 
 
 30 
       """
- 
 
 31 
        Documentation for constructor
- 
 
 32 
        initialize_table is called to make all the tables known in
- 
 
 33 
        the scope of this class.
- 
 
 34 
        Also start url needs to be feeded to the spider through start_urls.append
- 
 
 35 
        Domainname is name by which this spider is known outside
- 
 
 36 
        So this will be used as an argument for calling this spider.
- 
 
 37 
        As the number of pages to be crawled are not fixed so ct and no are used to make it dynamic.
- 
 
 38 
        """
26 
       initialize_table()
 39 
       initialize_table()
27 
       da = DataHelper()
 40 
       da = DataHelper()
28 
       #da.initxy()
 - 
 
29 
       #INFIBEAM_DOMAINNAME = "infibeam"
41 
       #INFIBEAM_DOMAINNAME = "infibeam"
30 
       INFIBEAM_DOMAINNAME = get_code_word("INFIBEAM_DOMAINNAME")
 42 
       INFIBEAM_DOMAINNAME = get_code_word("INFIBEAM_DOMAINNAME")
31 
       print INFIBEAM_DOMAINNAME
 43 
       print INFIBEAM_DOMAINNAME
32 
       self.domain_name = INFIBEAM_DOMAINNAME
 44 
       self.domain_name = INFIBEAM_DOMAINNAME
33 
       #INFIBEAM_CT = 15
 45 
       #INFIBEAM_CT = 15
Line 44... Line 56...
44 
            url1 = INFIBEAM_URL + str(no)
 56 
            url1 = INFIBEAM_URL + str(no)
45 
            self.start_urls.append(url1)
 57 
            self.start_urls.append(url1)
46 
            no=no+1
 58 
            no=no+1
47
59
48 
    def start_requests(self):
 60 
    def start_requests(self):
- 
 
 61 
        """
- 
 
 62 
        Documentation for method start_requests
- 
 
 63 
        To set various properties of the request to be made
- 
 
 64 
        like referer, headers and all.
- 
 
 65 
        Also suppliers entry need to be done in the table
- 
 
 66 
        datastore_datadefinition_suppliers.
- 
 
 67 
        @return a list of well formed requests which will be
- 
 
 68 
        crawled by spider and spider will return the response
- 
 
 69 
        """
49 
        #adding entry for the supplier i.e its name and site
 70 
        #adding entry for the supplier i.e its name and site
50 
        #INFIBEAM_HOMEPAGE = "www.infibeam.com"
 71 
        #INFIBEAM_HOMEPAGE = "www.infibeam.com"
51 
        INFIBEAM_HOMEPAGE = get_code_word("INFIBEAM_HOMEPAGE")
 72 
        INFIBEAM_HOMEPAGE = get_code_word("INFIBEAM_HOMEPAGE")
52 
        da = DataHelper()
 73 
        da = DataHelper()
53 
        da.add_supplier(self.domain_name, INFIBEAM_HOMEPAGE)
 74 
        da.add_supplier(self.domain_name, INFIBEAM_HOMEPAGE)
Line 61... Line 82...
61 
            request.headers.setdefault("Referer", INFIBEAM_REFERER)
 82 
            request.headers.setdefault("Referer", INFIBEAM_REFERER)
62 
            listreq.append(request)
83 
            listreq.append(request)
63 
        return listreq
 84 
        return listreq
64
85
65 
    def parse(self, response):
 86 
    def parse(self, response):
- 
 
 87 
        """
- 
 
 88 
        Documentation for method parse
- 
 
 89 
        @param response of individual requests
- 
 
 90 
        Using Xpaths needed information is extracted out of the response
- 
 
 91 
        and added to the database
- 
 
 92 
        Xpath1 = Give us section for individual phone
- 
 
 93 
        Xpath2 = Give us name for individual phone
- 
 
 94 
        Xpath3 = Give us quoted price for individual phone
- 
 
 95 
        vatplustax = to get final price from quoted price
- 
 
 96 
        Removelist = To filer the prices so as to make them integer for eg remove ',' or 'Rs'
- 
 
 97 
        """
66 
        da = DataHelper()
 98 
        da = DataHelper()
67 
        msg(response.url)
 99 
        msg(response.url)
68 
        #INFIBEAM_VATPLUSTAX = 0
 100 
        #INFIBEAM_VATPLUSTAX = 0
69 
        INFIBEAM_VATPLUSTAX = int(get_code_word("INFIBEAM_VATPLUSTAX"))
 101 
        INFIBEAM_VATPLUSTAX = int(get_code_word("INFIBEAM_VATPLUSTAX"))
70 
        #list elements are separated by ';'
 102 
        #list elements are separated by ';'