Subversion Repositories SmartDukaan

'''

Created on 28-May-2010

@author: gaurav

'''

from scrapy.spider import BaseSpider

from scrapy.selector import HtmlXPathSelector

from scrapy.http import Request

from demo.items import DemoItem

from scrapy.contrib.spidermiddleware import referer

from scrapy.http.headers import Headers

from scrapy.http.request.form import FormRequest

from scrapy.log import msg

from scrapy.http.response import Response

from time import *

from datastore import DataAccessor

from datastore.DataAccessor import DataHelper

import urllib

class naaptol_price2(BaseSpider):

    def __init__(self):

       self.domain_name = "naaptolphones2"

       #self.start_urls.append("http://www.naaptol.com/price/10415-Fly-Hummer-HT1.html")

       da = DataHelper()

       for pitem in da.get_allmorenaaptolurls():

            self.start_urls.append(pitem.url.strip())

    def start_requests(self):

        listreq = []

        for url1 in self.start_urls:

            request = Request(url = url1, callback=self.parse)

            request.headers.setdefault("Referer", "www.naaptol.com")

            listreq.append(request)

        return listreq

    def parse(self, response): 

        f = open('/home/gaurav/Desktop/response.txt', 'w')

        f.write(response.body)

        print "  url  " + response.url

        name = str(response.url)

        name_pos = name.rfind("/")

        name = name[name_pos+1:len(name)-5]

        hxs = HtmlXPathSelector(response)

        da = DataHelper()

       #prices = hxs.select('//table[@class ="ProductDetails"]/tbody/tr[6]/td/span/text()')

        prices = hxs.select('//table[@class ="ProductDetails"]//td[@class="Price"]/span/text()')

        try:

            price1 = prices.extract()[0]

            price1 = price1.decode("utf-8")

            price1 = price1.strip()

        except:

            price1 = ""

        try:

            price2 = prices.extract()[1]

            price2 = price2.decode("utf-8")

            price2 = price2.strip()

        except:

            price2 = ""

        try:

            if price1 == "" and price2 == "":

                prices = hxs.select('//table[@class ="ProductDetails"]//td[@class="Price"]/span/script/text()')

                price = str(prices.extract()[0])

                pos1 = price.find("'")

                pos2 = price.find("'",pos1+1,len(price))

                price1 = price[pos1+1:pos2] + "(approx)" 

                price2 = ""

        except:

            price1 = price2 = ""

        if price1 != '':

            price1 = price1.replace("Rs.", "")

            price1 = price1.replace(",", "")

            price1 = price1.strip()

        if price2 != '':        

            price2 = price2.replace("Rs.", "")

            price2 = price2.replace(",", "")

            price2 = price2.strip()

        if price1 == "Rates Not Available":

            price1 = price2 = ""

        print name

        print price1

        print price2

        print "\n"

        range = price1

        if price2 != "":

            range = str(range) + " to " 

            range = range + str(price2) 

            da.add_new_naaptolphone(name, range)

        OnlineSellers_pricelist = []

        OnlineSellers_namelist = []

        try:

            ct1 = hxs.select('//div[@id="OnlineSellers"]//div[@class="ProductResultHead"]//div[@class="headingstyle"]/text()')

            ct1 = str(ct1.extract()[0])

            ct1 = ct1.decode("utf-8")

            ct1 = ct1.strip()

            ps1 = ct1.find(" ")

            ct1 = ct1[0:ps1]

            ct1 = int(ct1)

        except:

            ct1 = 0

        ct = ct1

        i = 0

        os_info = hxs.select('//div[@id="onSellerContents"]//td[@class="price"]')

        #print len(os_info)

        while ct > 0:

            os = os_info[i].extract()

            ps1 = os.find(">")

            ps2 = os.find("<",ps1)

            os = os[ps1+1:ps2]

            os = os.replace("Rs.", "")

            os = os.replace(",", "")

            os = urllib.unquote(os)

            try:

                os = int(os)

            except:

                os = os_info[i].extract()

                ps1 = os.find(">",ps2)

                ps2 = os.find("<",ps1)

                os = os[ps1+1:ps2]

                os = os.replace("Rs.", "")

                os = os.replace(",", "")

                os = urllib.unquote(os)

                os = int(os) 

            print os

            OnlineSellers_pricelist.append(os)

            path = '//div[@id="onSellerContents"]//tr[@class="DottedBorder"]/td/a[@id="storeInfoPop'

            path = path + str(i) 

            path = path + '"]/span/text()'

            osname = hxs.select(path)

            #print len(osname)

            osname = osname.extract()[0]

            osname = urllib.unquote(osname)

            OnlineSellers_namelist.append(osname)

            print osname

            i = i+1

            ct = ct-1

        l = len(OnlineSellers_pricelist)

        i = 0

        nid = da.get_naaptolphone(name,range).id

        while l > 0:

            da.add_new_ntonlinesp(nid, OnlineSellers_namelist[i], OnlineSellers_pricelist[i])

            i = i+1

            l = l-1

        LocalSellers_pricelist = []   

        LocalSellers_namelist = []

        try:

            ct1 = hxs.select('//div[@id="LocalStores"]//div[@class="ProductResultHead"]//div[@class="headingstyle"]/text()')

            ct1 = str(ct1.extract()[0])

            ct1 = ct1.decode("utf-8")

            ct1 = ct1.strip()

            ps1 = ct1.find(" ")

            ct1 = ct1[0:ps1]

            ct1 = int(ct1)

        except:

            ct1 = 0

        ct = ct1

        i = 0

        os_info = hxs.select('//div[@id="offSellerContents"]//td[@class="price"]')

        os_names = hxs.select('//div[@id="offSellerContents"]//span[@class="LocalStoreHeading"]/text()')

        #print len(os_info)

        while ct > 0:

            os = os_info[i].extract()

            osname = os_names[i].extract() 

            #os = os.encode("utf-8")

            ps1 = os.find(">")

            ps2 = os.find("<",ps1)

            os = os[ps1+1:ps2]

            os = os.replace("Rs.", "")

            os = os.replace(",", "")

            os = urllib.unquote(os)

            osname = urllib.unquote(osname)

            try:

                os = int(os)

            except:

                os = os_info[i].extract()

                ps1 = os.find(">",ps2)

                ps2 = os.find("<",ps1)

                os = os[ps1+1:ps2]

                os = os.replace("Rs.", "")

                os = os.replace(",", "")

                os = urllib.unquote(os)

                os = int(os)        

            print os

            print osname

            LocalSellers_pricelist.append(os)

            LocalSellers_namelist.append(osname)

            i = i+1

            ct = ct-1

        l = len(LocalSellers_pricelist)

        i = 0

        nid = da.get_naaptolphone(name,range).id

        while l > 0:

            da.add_new_ntofflinesp(nid, LocalSellers_namelist[i], LocalSellers_pricelist[i])

            i = i+1

            l = l-1   

        '''

        l = len(OnlineSellers_list)

        i = 0

        while l > 0:

            #print OnlineSellers_list[i]

            i = i+1

            l = l-1 

        '''    

        f.close()

    '''    

        site = response.url

        vatplustax = 0

        pos1 = pos2 = 0

        temp = ""

        pos1 = site.rfind('/')

        if pos1 != -1:

            temp = site[pos1+1:len(site)]

            #pos2 = site.rfind('/',0,pos1-1)

        #if pos2 > 0:

            #temp = site[pos2+1:len(site)]

        pos3 = temp.find('.')

        temp1 = temp[pos3:len(temp)]

        name = temp.replace(temp1,"")         

        hxs = HtmlXPathSelector(response)

        prices = hxs.select('//div[@id ="priceComp"]//tr[2]/td[3]/span/text()')

        da = DataHelper()

        for price in prices:

             name = str(name).strip()

             price = price.extract()

             price = str(price).strip()

             price = price.replace("Rs", "")

             price = price.replace("/", "")

             price = price.replace("-", "")

             price = price.replace(".", "")

             shown_pr = int(price)

             final_pr = shown_pr + vatplustax

             da.add_new_mobstorephone(name,shown_pr,final_pr)

             print name

             print final_pr

             print "\n"

        for i in items:

            str1 = str(i['title']).strip() 

            print str1

            amnt = i['price'].replace(",","")

            amnt = amnt.replace("Rs", "")

            amnt = amnt.replace("/", "")

            amnt = amnt.replace("-", "")

            amnt = amnt.strip()

            vatplustax = 4*int(amnt)/100

            pr = int(amnt) + vatplustax 

            #print pr

            da.add_new_univerphone(str1,amnt,pr)

        '''        

SPIDER = naaptol_price2()