Subversion Repositories SmartDukaan

Rev

Rev 181 | Rev 259 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 181 Rev 228
Line 13... Line 13...
13 
from scrapy.http.headers import Headers
 13 
from scrapy.http.headers import Headers
14 
from scrapy.http.request.form import FormRequest
 14 
from scrapy.http.request.form import FormRequest
15 
from scrapy.log import msg
 15 
from scrapy.log import msg
16 
from scrapy.http.response import Response
 16 
from scrapy.http.response import Response
17 
 
 17 
 
18 
from datastore import DataAccessor
 18 
from datastore.DataAccessor import *
19 
from datastore.DataAccessor import DataHelper
 19 
from datastore.DataCodeAccessor import *
20 
 
 - 
 
- 
 
 20 
from html2text.unescaping import *
- 
 
 21 
from elixir import *
21 
 
 22 
 
22 
class infi_spider(BaseSpider):
 23 
class infi_spider(BaseSpider):
23
24
24 
    def __init__(self):
25 
    def __init__(self):
- 
 
 26 
       initialize_table()
- 
 
 27 
       da = DataHelper()
- 
 
 28 
       #da.initxy()
- 
 
 29 
       #INFIBEAM_DOMAINNAME = "infibeam"
- 
 
 30 
       INFIBEAM_DOMAINNAME = get_code_word("INFIBEAM_DOMAINNAME")
- 
 
 31 
       print INFIBEAM_DOMAINNAME
25 
       self.domain_name = "infibeam"
 32 
       self.domain_name = INFIBEAM_DOMAINNAME
- 
 
 33 
       #INFIBEAM_CT = 15
- 
 
 34 
       #INFIBEAM_CT = int(get_code_word("INFIBEAM_CT"))
26 
       ct = 15
 35 
       #ct = 15
- 
 
 36 
       ct = int(da.get_extra_vars('infibeam_count'))
- 
 
 37 
       #INFIBEAM_NO = 1
27 
       no = 1
 38 
       no = 1
- 
 
 39 
       if ct>14:
- 
 
 40 
           no = ct
- 
 
 41 
       #INFIBEAM_URL = "http://www.infibeam.com/Mobiles/search?page="
- 
 
 42 
       INFIBEAM_URL = get_code_word("INFIBEAM_URL")
28 
       while(no<=ct):
 43 
       while(no<=ct):
29 
            str1 = "http://www.infibeam.com/Mobiles/search?page="+str(no)
 44 
            url1 = INFIBEAM_URL + str(no)
30 
            self.start_urls.append(str1)
 45 
            self.start_urls.append(url1)
31 
            no=no+1
 46 
            no=no+1
32
47
33 
    def start_requests(self):
 48 
    def start_requests(self):
- 
 
 49 
        #adding entry for the supplier i.e its name and site
- 
 
 50 
        #INFIBEAM_HOMEPAGE = "www.infibeam.com"
- 
 
 51 
        INFIBEAM_HOMEPAGE = get_code_word("INFIBEAM_HOMEPAGE")
34 
        da = DataHelper()
 52 
        da = DataHelper()
35 
        da.add_supplier(self.domain_name, "www.infibeam.com")
 53 
        da.add_supplier(self.domain_name, INFIBEAM_HOMEPAGE)
36 
        listreq = []
 54 
        listreq = []
- 
 
 55
- 
 
 56 
        #for each request a referer has to be set
- 
 
 57 
        #INFIBEAM_REFERER = "www.google.com/search"
- 
 
 58 
        INFIBEAM_REFERER = get_code_word("INFIBEAM_REFERER")
37 
        for url1 in self.start_urls:
 59 
        for url1 in self.start_urls:
38 
            request = Request(url = str(url1), callback=self.parse,dont_filter=True,)
 60 
            request = Request(url = str(url1), callback=self.parse, dont_filter=True)
39 
            request.headers.setdefault("Referer", "http://www.infibeam.com/Mobiles")
 61 
            request.headers.setdefault("Referer", INFIBEAM_REFERER)
40 
            listreq.append(request)
 62 
            listreq.append(request)
41 
        return listreq
 63 
        return listreq
42
64
43 
    def parse(self, response):
 65 
    def parse(self, response):
44 
        vatplustax = 0
 66 
        da = DataHelper()
- 
 
 67 
        msg(response.url)
- 
 
 68 
        #INFIBEAM_VATPLUSTAX = 0
- 
 
 69 
        INFIBEAM_VATPLUSTAX = int(get_code_word("INFIBEAM_VATPLUSTAX"))
- 
 
 70 
        #list elements are separated by ';'
- 
 
 71 
        #INFIBEAM_REMOVELIST = ["Rs.",",","-","/","Rs"]
- 
 
 72 
        INFIBEAM_REMOVELIST = get_code_word("INFIBEAM_REMOVELIST")
- 
 
 73 
        INFIBEAM_REMOVELIST = INFIBEAM_REMOVELIST.split(';')
45 
        hxs = HtmlXPathSelector(response)
 74 
        hxs = HtmlXPathSelector(response)
46 
        phone_info = hxs.select('//ul[@class="srch_result portrait"]/li')
 75 
        #INFIBEAM_XPATH1 = '//ul[@class="srch_result portrait"]/li'
- 
 
 76 
        INFIBEAM_XPATH1 = get_code_word("INFIBEAM_XPATH1")
- 
 
 77 
        phone_info = hxs.select(INFIBEAM_XPATH1)
- 
 
 78 
        #INFIBEAM_XPATH2 = './/p/span[@class="title"]/text()'
- 
 
 79 
        INFIBEAM_XPATH2 = get_code_word("INFIBEAM_XPATH2")
- 
 
 80 
        #INFIBEAM_XPATH3 = './/p/span[@class="price"]/text()'
- 
 
 81 
        INFIBEAM_XPATH3 = get_code_word("INFIBEAM_XPATH3")
47 
        items = []
 82 
        items = []
48 
        msg(response.url)
 83 
        msg(response.url)
- 
 
 84
49 
        for i in phone_info:
 85 
        if not phone_info:
- 
 
 86 
            ct = int(da.get_extra_vars('infibeam_count'))
50 
            item = {}
 87 
            if ct>14:
51 
            item['name'] = i.select('.//p/span[@class="title"]/text()')[0].extract()
 - 
 
52 
            item['price'] = i.select('.//p/span[@class="price"]/text()')[0].extract()
 88 
                fails = int(da.get_extra_vars('infibeam_fails'))
53 
            items.append(item)
 89 
                fails = fails+1
54
90 
                if fails > 0:
55 
        f = open('/home/gaurav/Desktop/infibeam_info.txt', 'a')
 91 
                    da.set_extra_vars('infibeam_flag','FALSE','')
- 
 
 92 
                da.set_extra_vars('infibeam_fails',str(fails),'')
56 
        da = DataHelper()
 93 
        else:
- 
 
 94 
            for i in phone_info:
57 
        for item in items:
 95 
                item = {}
58 
            amnt = item['price'].replace(",","")
 96 
                item['name'] = i.select(INFIBEAM_XPATH2)[0].extract()
59 
            amnt = amnt.replace("Rs.", "")
 97 
                item['price'] = i.select(INFIBEAM_XPATH3)[0].extract()
60 
            amnt = amnt.strip()
 98 
                items.append(item)
61
99
62 
            pr = int(amnt) + vatplustax
 100 
            for i in items:
63 
            da.add_infiphone(item['name'], amnt,pr)
 101 
                amnt = i['price']
64 
            #print item['name']
 102 
                if amnt != '':
65 
            #print item['price']
 103 
                    for r in INFIBEAM_REMOVELIST:
66 
            f.write(item['name'])
 104 
                        while amnt.find(r) != -1:
67 
            f.write("\n")
 105 
                            amnt = amnt.replace(r, "")
68 
            f.write(str(amnt))
 106 
                        amnt = amnt.strip()
69 
            f.write("\n")
 107 
                pr = int(amnt) + INFIBEAM_VATPLUSTAX
70 
            f.write(str(pr))
 108 
                da.add_infiphone(i['name'], amnt,pr)
71 
            f.write("\n\n\n")
 109
72 
        f.close()
110
73 
SPIDER = infi_spider()
 111 
SPIDER = infi_spider()
74 

            112