Subversion Repositories SmartDukaan

Rev

Rev 233 | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 233 Rev 260
Line 19... Line 19...
19 
 
 19 
 
20 
from html2text.unescaping import *
 20 
from html2text.unescaping import *
21 
import urllib
 21 
import urllib
22 
 
 22 
 
23 
class mobilestore_spider0(BaseSpider):
 23 
class mobilestore_spider0(BaseSpider):
- 
 
 24 
    """
- 
 
 25 
    Documentation for class mobilestore_spider0
- 
 
 26 
    This spider collects the information for the individual phones
- 
 
 27 
    and store them in table datastore_datadefinition_themobilestorephones_new
- 
 
 28 
    """
24
29
25 
    def __init__(self):
 30 
    def __init__(self):
- 
 
 31 
       """
- 
 
 32 
        Documentation for constructor
- 
 
 33 
        initialize_table is called to make all the tables known in
- 
 
 34 
        the scope of this class.
- 
 
 35 
        Also start url needs to be feeded to the spider through start_urls.append
- 
 
 36 
        Domainname is name by which this spider is known outside
- 
 
 37 
        So this will be used as an argument for calling this spider.
- 
 
 38 
        As the number of pages to be crawled are not fixed so ct and nt are used to make it dynamic.
- 
 
 39 
        url1 and url2 are used for getting actual start urls.
- 
 
 40 
       """
26 
       initialize_table()
41 
       initialize_table()
27 
       #MOBILESTORE_DOMAINNAME0 = "mobilestore0"
42 
       #MOBILESTORE_DOMAINNAME0 = "mobilestore0"
28 
       MOBILESTORE_DOMAINNAME0 = get_code_word("MOBILESTORE_DOMAINNAME0")
 43 
       MOBILESTORE_DOMAINNAME0 = get_code_word("MOBILESTORE_DOMAINNAME0")
29 
       self.domain_name = MOBILESTORE_DOMAINNAME0
 44 
       self.domain_name = MOBILESTORE_DOMAINNAME0
30 
       da = DataHelper()
 45 
       da = DataHelper()
31 
       MOBILESTORE_CT = int(da.get_extra_vars('mobilestore_count'))
 46 
       MOBILESTORE_CT = int(da.get_extra_vars('mobilestore_count'))
32 
       nt = 1
 47 
       nt = 1
33 
       if MOBILESTORE_CT > 800:
 48 
       if MOBILESTORE_CT > 800:
34 
           nt = MOBILESTORE_CT-50
 49 
           nt = MOBILESTORE_CT-50
35 
       while nt < MOBILESTORE_CT:
 50 
       while nt < MOBILESTORE_CT:
36 
           #MOBILESTORE_URL2 = "http://www.themobilestore.in/mobilestore/faces/tiles/product.jsp?productID=600&catalogueID=3"
 - 
 
37 
           #MOBILESTORE_URL1 = "http://www.themobilestore.in/mobilestore/faces/tiles/product.jsp?productID=" + str(nt)
 51 
           #MOBILESTORE_URL1 = "http://www.themobilestore.in/mobilestore/faces/tiles/product.jsp?productID=" + str(nt)
38 
           MOBILESTORE_URL1 = get_code_word("MOBILESTORE_URL1") + str(nt)
 52 
           MOBILESTORE_URL1 = get_code_word("MOBILESTORE_URL1") + str(nt)
39 
           #MOBILESTORE_URL2 = MOBILESTORE_URL1 + "&catalogueID=3"
 53 
           #MOBILESTORE_URL2 = MOBILESTORE_URL1 + "&catalogueID=3"
40 
           MOBILESTORE_URL2 = MOBILESTORE_URL1 + get_code_word("MOBILESTORE_URL2")
 54 
           MOBILESTORE_URL2 = MOBILESTORE_URL1 + get_code_word("MOBILESTORE_URL2")
41 
           self.start_urls.append(MOBILESTORE_URL2)
 55 
           self.start_urls.append(MOBILESTORE_URL2)
42 
           nt = nt+1
 56 
           nt = nt+1
43
57
44 
    def start_requests(self):
 58 
    def start_requests(self):
- 
 
 59 
        """
- 
 
 60 
        Documentation for method start_requests
- 
 
 61 
        To set various properties of the request to be made
- 
 
 62 
        like referer, headers and all.
- 
 
 63 
        Also suppliers entry need to be done in the table
- 
 
 64 
        datastore_datadefinition_suppliers.
- 
 
 65 
        @return a list of well formed requests which will be
- 
 
 66 
        crawled by spider and spider will return the response
- 
 
 67 
        """
45 
         #adding entry for the supplier i.e its name and site
 68 
        #adding entry for the supplier i.e its name and site
46 
        #MOBILESTORE_HOMEPAGE = "www.themobilestore.in"
 69 
        #MOBILESTORE_HOMEPAGE = "www.themobilestore.in"
47 
        MOBILESTORE_HOMEPAGE = get_code_word("MOBILESTORE_HOMEPAGE")
 70 
        MOBILESTORE_HOMEPAGE = get_code_word("MOBILESTORE_HOMEPAGE")
48 
        da = DataHelper()
 71 
        da = DataHelper()
49 
        da.add_supplier(self.domain_name, MOBILESTORE_HOMEPAGE)
 72 
        da.add_supplier(self.domain_name, MOBILESTORE_HOMEPAGE)
50 
        listreq = []
 73 
        listreq = []
Line 56... Line 79...
56 
            request.headers.setdefault("Referer", MOBILESTORE_REFERER)
 79 
            request.headers.setdefault("Referer", MOBILESTORE_REFERER)
57 
            listreq.append(request)
 80 
            listreq.append(request)
58 
        return listreq
 81 
        return listreq
59
82
60 
    def parse(self, response):
 83 
    def parse(self, response):
- 
 
 84 
        """
- 
 
 85 
        Documentation for method parse
- 
 
 86 
        @param response of individual requests
- 
 
 87 
        Using Xpaths needed information is extracted out of the response
- 
 
 88 
        and added to the database
- 
 
 89 
        Xpath3 = Give us name for individual phone
- 
 
 90 
        Xpath4 = Give us price for individual phone
- 
 
 91 
        Xpath5 = Give us name for individual phone, if its not gettable from xpath3
- 
 
 92 
        Xpath6 = Give us name for individual phone, if its not gettable from xpath3 and xpath5
- 
 
 93 
        Xpath7 = to check that the phone can be bought or not
- 
 
 94 
        Xpath8 = to check that the item is mobile phone
- 
 
 95 
        """
61 
        da = DataHelper()
 96 
        da = DataHelper()
62 
        #file_to_write1 = "/tmp/phones.csv"
 - 
 
63 
        #data_file1 = open(file_to_write1,"a")
 - 
 
64 
        url1 = response.url
 97 
        url1 = response.url
65 
        ps1 = url1.find('=')
 98 
        ps1 = url1.find('=')
66 
        ps2 = url1.find('&')
 99 
        ps2 = url1.find('&')
67 
        str1 = url1[ps1+1:ps2]
 100 
        str1 = url1[ps1+1:ps2]
68 
        #da = DataHelper()
 101 
        #da = DataHelper()
Line 98... Line 131...
98 
                    name = name.strip()
 131 
                    name = name.strip()
99 
                    price = hxs.select(MOBILESTORE_XPATH4)
 132 
                    price = hxs.select(MOBILESTORE_XPATH4)
100 
                    price = price[0].extract()
 133 
                    price = price[0].extract()
101 
                    price = price.strip()
 134 
                    price = price.strip()
102 
                    price = int(price)
 135 
                    price = int(price)
103 
                    da.add_new_mobstorephone_new(name, price, price, str2)
 136 
                    da.add_new_mobstorephone_new(name, price, price, str2)
104 
                    #csv_data1 =  "%s, %s, %s, %s" %(name, price, str1, str2)
 - 
 
105 
                    #data_file1.write(csv_data1)
 - 
 
106 
                    #data_file1.write("\n")
 - 
 
107 
                    print name
 - 
 
108 
                    print price
- 
 
109 
                except:
 137 
                except:
110 
                    try:
 138 
                    try:
111 
                        name = hxs.select(MOBILESTORE_XPATH5)
 139 
                        name = hxs.select(MOBILESTORE_XPATH5)
112 
                        name = name[0].extract()
 140 
                        name = name[0].extract()
113 
                        name = name.strip()
 141 
                        name = name.strip()
114 
                        price = hxs.select(MOBILESTORE_XPATH4)
 142 
                        price = hxs.select(MOBILESTORE_XPATH4)
115 
                        price = price[0].extract()
 143 
                        price = price[0].extract()
116 
                        price = price.strip()
 144 
                        price = price.strip()
117 
                        price = int(price)
 145 
                        price = int(price)
118 
                        da.add_new_mobstorephone_new(name, price, price, str2)
 146 
                        da.add_new_mobstorephone_new(name, price, price, str2)
119 
                        #csv_data1 =  "%s, %s, %s, %s" %(name, price, str1, str2)
 - 
 
120 
                        #data_file1.write(csv_data1)
 - 
 
121 
                        #data_file1.write("\n")
 - 
 
122 
                        print name
 - 
 
123 
                        print price
- 
 
124 
                    except:
 147 
                    except:
125 
                        try:
 148 
                        try:
126 
                            name = hxs.select(MOBILESTORE_XPATH6)
 149 
                            name = hxs.select(MOBILESTORE_XPATH6)
127 
                            name = name[0].extract()
 150 
                            name = name[0].extract()
128 
                            name = name.strip()
 151 
                            name = name.strip()
129 
                            price = hxs.select(MOBILESTORE_XPATH4)
 152 
                            price = hxs.select(MOBILESTORE_XPATH4)
130 
                            price = price[0].extract()
 153 
                            price = price[0].extract()
131 
                            price = price.strip()
 154 
                            price = price.strip()
132 
                            price = int(price)
 155 
                            price = int(price)
133 
                            da.add_new_mobstorephone_new(name, price, price, str2)
 156 
                            da.add_new_mobstorephone_new(name, price, price, str2)
134 
                            #csv_data1 =  "%s, %s, %s, %s" %(name, price, str1, str2)
 - 
 
135 
                            #data_file1.write(csv_data1)
 - 
 
136 
                            #data_file1.write("\n")
 - 
 
137 
                            print name
 - 
 
138 
                            print price
- 
 
139 
                        except:
 157 
                        except:
140 
                            #csv_data2 =  "%s" %(str1)
 - 
 
141 
                            #data_file2.write(csv_data2)
 - 
 
142 
                            #data_file2.write("\n")
- 
 
143 
                            print str1
 158 
                            pass
144 
        except:
159 
        except:
145 
          ct = int(da.get_extra_vars('mobilestore_count'))
 160 
          ct = int(da.get_extra_vars('mobilestore_count'))
146 
          if ct>800:
 161 
          if ct>800:
147 
            fails = int(da.get_extra_vars('mobilestore_fails'))
 162 
            fails = int(da.get_extra_vars('mobilestore_fails'))
148 
            fails = fails+1
 163 
            fails = fails+1
149 
            da.set_extra_vars('mobilestore_fails',str(fails),'')
 164 
            da.set_extra_vars('mobilestore_fails',str(fails),'')
150 
            if fails > 40:
 165 
            if fails > 40:
151 
                da.set_extra_vars('mobilestore_flag','FALSE','')
 166 
                da.set_extra_vars('mobilestore_flag','FALSE','')
152
167
153 
          print " something else"
 - 
 
154 
        #data_file1.close()
 - 
 
155 
        #data_file2.close()
- 
 
156 
SPIDER = mobilestore_spider0()
 168 
SPIDER = mobilestore_spider0()
157 

            169