WebSVN – SmartDukaan – Diff – /prototype/naaptolpass2/src/demo/spiders/spider2.py

 from scrapy.http.headers import Headers
 from scrapy.http.request.form import FormRequest
 from scrapy.log import msg
 from scrapy.http.response import Response
 from time import *
+from datastore.DataCodeAccessor import *
+from datastore.DataAccessor import *
-from datastore import DataAccessor
-from datastore.DataAccessor import DataHelper
 import urllib
+from html2text.unescaping import *
 class naaptol_price(BaseSpider):
     def __init__(self):
+       initialize_table()
-       self.domain_name = "naaptolphones"
+       #NAAPTOL_DOMAINNAME1 = "naaptol1"
-       #self.start_urls.append("http://www.naaptol.com/price/10415-Fly-Hummer-HT1.html")
+       NAAPTOL_DOMAINNAME1 = get_code_word("NAAPTOL_DOMAINNAME1")
+       self.domain_name = NAAPTOL_DOMAINNAME1
+       # get urls from the database and append them in the list for crawling
        da = DataHelper()
+       #url = "http://www.naaptol.com/features/10417-Fly-E300.html"
+       #self.start_urls.append(url)
        for pitem in da.get_allnaaptolurls():
             self.start_urls.append(pitem.url.strip())
     def start_requests(self):
+        #for each request a referer has to be set
         listreq = []
+        #NAAPTOL_REFERER = "http://www.google.com"
+        NAAPTOL_REFERER = get_code_word("NAAPTOL_REFERER")
         for url1 in self.start_urls:
-            request = Request(url = url1, callback=self.parse)
+            request = Request(url = str(url1), callback=self.parse)
-            request.headers.setdefault("Referer", "www.naaptol.com")
+            request.headers.setdefault("Referer", NAAPTOL_REFERER)
             listreq.append(request)
         return listreq
     def parse(self, response):
+        # there are two different type of urls one contains feature and other one contains price
+        #both have to be processed differently
-       #msg(response.body)
+        msg(response.url)
         site = response.url
+        site = unescape(site)
         sp1 = site.rfind("/")
         sp2 = site.rfind("/",0,sp1-1)
         catg = site[sp2+1:sp1]
         da = DataHelper()
+        #change price to features and add to urls as both provide the same data but in different formats
+        #otherwise crawl the url containing features
+        #NAAPTOL_CHKLIST2 = ['price']
+        #list separated by ';'
+        NAAPTOL_CHKLIST2 = get_code_word("NAAPTOL_CHKLIST2")
+        NAAPTOL_CHKLIST2 = NAAPTOL_CHKLIST2.split(';')
+        #NAAPTOL_PART = "features"
+        NAAPTOL_PART = get_code_word("NAAPTOL_PART")
+        #NAAPTOL_REMOVELIST = ["Rs.",","]
+        NAAPTOL_REMOVELIST = get_code_word("NAAPTOL_REMOVELIST")
+        NAAPTOL_REMOVELIST = NAAPTOL_REMOVELIST.split(';')
+        for c in NAAPTOL_CHKLIST2:
-        if catg == "price":
+            if c == catg:
-            site = site.replace("price","features")
+                site = site.replace(c,NAAPTOL_PART)
-            da.add_morenaaptolurl(site)
+                da.add_morenaaptolurl(site)
-        else:
-            f = open('/home/gaurav/Desktop/response.txt', 'w')
-            f.write(response.body)
+        if catg == NAAPTOL_PART:
-            print "  url  " + response.url
+            #retreiving name from the the url
             name = str(response.url)
+            name = unescape(name)
             name_pos = name.rfind("/")
             name = name[name_pos+1:len(name)-5]
+            name_pos = name.find("-")
+            name = name[name_pos+1:len(name)]
             hxs = HtmlXPathSelector(response)
-           #prices = hxs.select('//table[@class ="ProductDetails"]/tbody/tr[6]/td/span/text()')
+            #price and price2 determine range
-            prices = hxs.select('//table[@class ="ProductDetails"]//td[@class="Price"]/span/text()')
+            #NAAPTOL_XPATH2 = '//table[@class ="ProductDetails"]//td[@class="Price"]/span/text()'
+            NAAPTOL_XPATH2 = get_code_word("NAAPTOL_XPATH2")
+            prices = hxs.select(NAAPTOL_XPATH2)
             try:
                 price1 = prices.extract()[0]
-                price1 = price1.decode("utf-8")
+                #price1 = price1.decode("utf-8")
                 price1 = price1.strip()
             except:
                 price1 = ""
             try:
                 price2 = prices.extract()[1]
-                price2 = price2.decode("utf-8")
+                #price2 = price2.decode("utf-8")
                 price2 = price2.strip()
             except:
                 price2 = ""
             try:
                 if price1 == "" and price2 == "":
-                    prices = hxs.select('//table[@class ="ProductDetails"]//td[@class="Price"]/span/script/text()')
+                    #NAAPTOL_XPATH3 = '//table[@class ="ProductDetails"]//td[@class="Price"]/span/script/text()'
+                    NAAPTOL_XPATH3 = get_code_word("NAAPTOL_XPATH3")
+                    prices = hxs.select(NAAPTOL_XPATH3)
                     price = str(prices.extract()[0])
                     pos1 = price.find("'")
                     pos2 = price.find("'",pos1+1,len(price))
                     price1 = price[pos1+1:pos2] + "(approx)"
                     price2 = ""
             except:
                 price1 = price2 = ""
+            #removelist is used for converting price to decimal format containing only numbers and '.'
             if price1 != '':
+                for r in NAAPTOL_REMOVELIST:
-                price1 = price1.replace("Rs.", "")
+                    while price1.find(r) != -1:
-                price1 = price1.replace(",", "")
+                        price1 = price1.replace(r, "")
                 price1 = price1.strip()
             if price2 != '':
+                for r in NAAPTOL_REMOVELIST:
-                price2 = price2.replace("Rs.", "")
+                    while price2.find(r) != -1:
-                price2 = price2.replace(",", "")
+                        price2 = price2.replace(r, "")
                 price2 = price2.strip()
             if price1 == "Rates Not Available":
                 price1 = price2 = ""
-            print name
-            print price1
-            print price2
+            #range = price1 to price2
-            print "\n"
             range = price1
             if price2 != "":
                 range = str(range) + " to "
                 range = range + str(price2)
-            da.add_new_naaptolphone(name, range)
+            da.add_new_naaptolphone(name, str(range))
             OnlineSellers_pricelist = []
             OnlineSellers_namelist = []
             try:
+                #ct1 holds the count of online sellers
-                ct1 = hxs.select('//div[@id="OnlineSellers"]//div[@class="ProductResultHead"]//div[@class="headingstyle"]/text()')
+                #NAAPTOL_XPATH4 = '//div[@id="OnlineSellers"]//div[@class="ProductResultHead"]//div[@class="headingstyle"]/text()'
+                NAAPTOL_XPATH4 = get_code_word("NAAPTOL_XPATH4")
+                ct1 = hxs.select(NAAPTOL_XPATH4)
                 ct1 = str(ct1.extract()[0])
                 ct1 = ct1.decode("utf-8")
                 ct1 = ct1.strip()
                 ps1 = ct1.find(" ")
                 ct1 = ct1[0:ps1]
                 ct1 = int(ct1)
             except:
                 ct1 = 0
             ct = ct1
             i = 0
-            os_info = hxs.select('//div[@id="onSellerContents"]//td[@class="price"]')
+            #NAAPTOL_XPATH5 = '//div[@id="onSellerContents"]//td[@class="price"]'
+            NAAPTOL_XPATH5 = get_code_word("NAAPTOL_XPATH5")
-            #print len(os_info)
+            os_info = hxs.select(NAAPTOL_XPATH5)
             while ct > 0:
                 os = os_info[i].extract()
                 ps1 = os.find(">")
                 ps2 = os.find("<",ps1)
                 os = os[ps1+1:ps2]
-                os = os.replace("Rs.", "")
+                if os != '':
+                    for r in NAAPTOL_REMOVELIST:
+                        while os.find(r) != -1:
-                os = os.replace(",", "")
+                            os = os.replace(r, "")
                 os = urllib.unquote(os)
                 try:
                     os = int(os)
                 except:
+                    #stored in format different than previous one
                     os = os_info[i].extract()
                     ps1 = os.find(">",ps2)
                     ps2 = os.find("<",ps1)
                     os = os[ps1+1:ps2]
-                    os = os.replace("Rs.", "")
+                    if os != '':
+                        for r in NAAPTOL_REMOVELIST:
+                            while os.find(r) != -1:
-                    os = os.replace(",", "")
+                                os = os.replace(r, "")
                     os = urllib.unquote(os)
                     os = int(os)
-                print os
                 OnlineSellers_pricelist.append(os)
-                path = '//div[@id="onSellerContents"]//tr[@class="DottedBorder"]/td/a[@id="storeInfoPop'
+                #NAAPTOL_XPATH6 = '//div[@id="onSellerContents"]//tr[@class="DottedBorder"]/td/a[@id="storeInfoPop'
+                NAAPTOL_XPATH6 = get_code_word("NAAPTOL_XPATH6")
+                #NAAPTOL_XPATH7 = '"]/span/text()'
+                NAAPTOL_XPATH7 = get_code_word("NAAPTOL_XPATH7")
-                path = path + str(i)
+                NAAPTOL_XPATH6 = NAAPTOL_XPATH6 + str(i)
+                NAAPTOL_XPATH6 = NAAPTOL_XPATH6 + NAAPTOL_XPATH7
-                path = path + '"]/span/text()'
+                path = NAAPTOL_XPATH6
                 osname = hxs.select(path)
-                #print len(osname)
                 osname = osname.extract()[0]
+                osname = unescape(osname)
                 osname = urllib.unquote(osname)
                 OnlineSellers_namelist.append(osname)
-                print osname
                 i = i+1
                 ct = ct-1
             l = len(OnlineSellers_pricelist)
             i = 0
             nid = da.get_naaptolphone(name,range).id
             while l > 0:
                 da.add_new_ntonlinesp(nid, OnlineSellers_namelist[i], OnlineSellers_pricelist[i])
-                #print OnlineSellers_list[i]
                 i = i+1
                 l = l-1
             LocalSellers_pricelist = []
             LocalSellers_namelist = []
             try:
+                #ct1 holds the count of online sellers
-                ct1 = hxs.select('//div[@id="LocalStores"]//div[@class="ProductResultHead"]//div[@class="headingstyle"]/text()')
+                #NAAPTOL_XPATH8 = '//div[@id="LocalStores"]//div[@class="ProductResultHead"]//div[@class="headingstyle"]/text()'
+                NAAPTOL_XPATH8 = get_code_word("NAAPTOL_XPATH8")
+                ct1 = hxs.select(NAAPTOL_XPATH8)
                 ct1 = str(ct1.extract()[0])
                 ct1 = ct1.decode("utf-8")
                 ct1 = ct1.strip()
                 ps1 = ct1.find(" ")
                 ct1 = ct1[0:ps1]
                 ct1 = int(ct1)
             except:
                 ct1 = 0
             ct = ct1
             i = 0
-            os_info = hxs.select('//div[@id="offSellerContents"]//td[@class="price"]')
+            #NAAPTOL_XPATH9 = '//div[@id="offSellerContents"]//td[@class="price"]'
+            NAAPTOL_XPATH9 = get_code_word("NAAPTOL_XPATH9")
-            os_names = hxs.select('//div[@id="offSellerContents"]//span[@class="LocalStoreHeading"]/text()')
+            #NAAPTOL_XPATH10 = '//div[@id="offSellerContents"]//span[@class="LocalStoreHeading"]/text()'
+            NAAPTOL_XPATH10 = get_code_word("NAAPTOL_XPATH10")
-            #print len(os_info)
+            os_info = hxs.select(NAAPTOL_XPATH9)
+            os_names = hxs.select(NAAPTOL_XPATH10)
             while ct > 0:
                 os = os_info[i].extract()
                 osname = os_names[i].extract()
-                #os = os.encode("utf-8")
                 ps1 = os.find(">")
                 ps2 = os.find("<",ps1)
                 os = os[ps1+1:ps2]
-                os = os.replace("Rs.", "")
+                if os != '':
+                    for r in NAAPTOL_REMOVELIST:
+                        while os.find(r) != -1:
-                os = os.replace(",", "")
+                            os = os.replace(r, "")
                 os = urllib.unquote(os)
                 osname = urllib.unquote(osname)
+                osname = unescape(osname)
                 try:
                     os = int(os)
                 except:
+                    #stored in format different than previous one
                     os = os_info[i].extract()
                     ps1 = os.find(">",ps2)
                     ps2 = os.find("<",ps1)
                     os = os[ps1+1:ps2]
-                    os = os.replace("Rs.", "")
+                    if os != '':
+                        for r in NAAPTOL_REMOVELIST:
+                            while os.find(r) != -1:
-                    os = os.replace(",", "")
+                                os = os.replace(r, "")
                     os = urllib.unquote(os)
                     os = int(os)
-                print os
-                print osname
                 LocalSellers_pricelist.append(os)
                 LocalSellers_namelist.append(osname)
                 i = i+1
                 ct = ct-1
             l = len(LocalSellers_pricelist)
             i = 0
             nid = da.get_naaptolphone(name,range).id
             while l > 0:
                 da.add_new_ntofflinesp(nid, LocalSellers_namelist[i], LocalSellers_pricelist[i])
                 i = i+1
                 l = l-1
-            '''
-            l = len(OnlineSellers_list)
-            i = 0
-            while l > 0:
-                #print OnlineSellers_list[i]
-                i = i+1
-                l = l-1
-            '''
-            f.close()
-            #del DataHelper
-        '''
-            site = response.url
-            vatplustax = 0
-            pos1 = pos2 = 0
-            temp = ""
-            pos1 = site.rfind('/')
-            if pos1 != -1:
-                temp = site[pos1+1:len(site)]
-                #pos2 = site.rfind('/',0,pos1-1)
-            #if pos2 > 0:
-                #temp = site[pos2+1:len(site)]
-            pos3 = temp.find('.')
-            temp1 = temp[pos3:len(temp)]
-            name = temp.replace(temp1,"")
-            hxs = HtmlXPathSelector(response)
-            prices = hxs.select('//div[@id ="priceComp"]//tr[2]/td[3]/span/text()')
-            da = DataHelper()
-            for price in prices:
-                 name = str(name).strip()
-                 price = price.extract()
-                 price = str(price).strip()
-                 price = price.replace("Rs", "")
-                 price = price.replace("/", "")
-                 price = price.replace("-", "")
-                 price = price.replace(".", "")
-                 shown_pr = int(price)
-                 final_pr = shown_pr + vatplustax
-                 da.add_new_mobstorephone(name,shown_pr,final_pr)
-                 print name
-                 print final_pr
-                 print "\n"
-            for i in items:
-                str1 = str(i['title']).strip()
-                print str1
-                amnt = i['price'].replace(",","")
-                amnt = amnt.replace("Rs", "")
-                amnt = amnt.replace("/", "")
-                amnt = amnt.replace("-", "")
-                amnt = amnt.strip()
-                vatplustax = 4*int(amnt)/100
-                pr = int(amnt) + vatplustax
-                #print pr
-                da.add_new_univerphone(str1,amnt,pr)
-            '''
 SPIDER = naaptol_price()

Subversion Repositories SmartDukaan

(root)/prototype/naaptolpass2/src/demo/spiders/spider2.py – Rev 189 → 236