| 1336 |
rajveer |
1 |
import os
|
|
|
2 |
import subprocess
|
|
|
3 |
import sys
|
|
|
4 |
from datastore.DataAccessor import DataHelper
|
|
|
5 |
|
|
|
6 |
"""
|
|
|
7 |
Documentation for script runinfibeam.py
|
|
|
8 |
This is the script called by consetup.py
|
|
|
9 |
First it will run the spider for infibeam dynamically (i.e for determining no of pages)
|
|
|
10 |
Then, it will generate the csv file
|
|
|
11 |
@param path to the folder in which spider-projects reside (:/home/gaurav/code) but start with pathsep
|
|
|
12 |
"""
|
|
|
13 |
|
|
|
14 |
ps = os.pathsep
|
|
|
15 |
ds = os.sep
|
|
|
16 |
path = sys.argv[1]
|
|
|
17 |
path0 = path[1:len(path)]+ds+"ScrapyDatastore"+ds+"src"
|
|
|
18 |
path1 = path+ds+"ExtraFunctions"+ds+"src"
|
|
|
19 |
path2 = path+ds+"filterforname"+ds+"src"
|
|
|
20 |
path3 = path0+path1+path2
|
|
|
21 |
os.environ['PYTHONPATH'] = path3
|
|
|
22 |
|
|
|
23 |
path3 = str(os.getenv("PYTHONPATH"))
|
|
|
24 |
path4 = path+ds+"infibeamScrapy"+ds+"src"
|
|
|
25 |
path3 = path3 + path4
|
|
|
26 |
os.putenv("PYTHONPATH",path3)
|
|
|
27 |
path4 = path4.replace(ps,"")
|
|
|
28 |
path4 = path4+ds+"scrapy-ctl.py"
|
|
|
29 |
|
|
|
30 |
da = DataHelper()
|
|
|
31 |
da.initxy()
|
|
|
32 |
#da = DataHelper()
|
|
|
33 |
ct = 14
|
|
|
34 |
da.set_extra_vars('infibeam_count',str(ct),'')
|
|
|
35 |
da.set_extra_vars('infibeam_fails','0','')
|
|
|
36 |
da.set_extra_vars('infibeam_flag','TRUE','')
|
|
|
37 |
|
|
|
38 |
flag = da.get_extra_vars('infibeam_flag')
|
|
|
39 |
while flag=='TRUE':
|
|
|
40 |
subprocess.call(["python", path4, "crawl", "infibeam"])
|
|
|
41 |
ct = ct+1
|
|
|
42 |
da.set_extra_vars('infibeam_count',str(ct),'')
|
|
|
43 |
flag = da.get_extra_vars('infibeam_flag')
|
|
|
44 |
|