| 1336 |
rajveer |
1 |
import os
|
|
|
2 |
import subprocess
|
|
|
3 |
from datastore.DataAccessor import DataHelper
|
|
|
4 |
import sys
|
|
|
5 |
|
|
|
6 |
"""
|
|
|
7 |
Documentation for script runmobstore.py
|
|
|
8 |
This is the script called by consetup.py
|
|
|
9 |
First it will run the spider for mobilestore dynamically (i.e for determining no of pages)
|
|
|
10 |
Then, it will generate the csv file
|
|
|
11 |
@param path to the folder in which spider-projects reside (:/home/gaurav/code) but start with pathsep
|
|
|
12 |
"""
|
|
|
13 |
ps = os.pathsep
|
|
|
14 |
ds = os.sep
|
|
|
15 |
path = sys.argv[1]
|
|
|
16 |
path0 = path[1:len(path)]+ds+"ScrapyDatastore"+ds+"src"
|
|
|
17 |
path1 = path+ds+"ExtraFunctions"+ds+"src"
|
|
|
18 |
path2 = path+ds+"filterforname"+ds+"src"
|
|
|
19 |
path3 = path0+path1+path2
|
|
|
20 |
os.environ['PYTHONPATH'] = path3
|
|
|
21 |
|
|
|
22 |
path3 = str(os.getenv("PYTHONPATH"))
|
|
|
23 |
path4 = path + ds+"mobilestoreScrapy0"+ds+"src"
|
|
|
24 |
path3 = path3 + path4
|
|
|
25 |
os.putenv("PYTHONPATH",path3)
|
|
|
26 |
path4 = path4.replace(ps,"")
|
|
|
27 |
path4 = path4 + ds+"scrapy-ctl.py"
|
|
|
28 |
|
|
|
29 |
da = DataHelper()
|
|
|
30 |
da.initxy()
|
|
|
31 |
#da = DataHelper()
|
|
|
32 |
ct = 800
|
|
|
33 |
da.set_extra_vars('mobilestore_count',str(ct),'')
|
|
|
34 |
da.set_extra_vars('mobilestore_fails','0','')
|
|
|
35 |
da.set_extra_vars('mobilestore_flag','TRUE','')
|
|
|
36 |
|
|
|
37 |
flag = da.get_extra_vars('mobilestore_flag')
|
|
|
38 |
print flag
|
|
|
39 |
|
|
|
40 |
while flag=='TRUE':
|
|
|
41 |
subprocess.call(["python", path4, "crawl", "mobilestore0"])
|
|
|
42 |
ct = ct+50
|
|
|
43 |
da.set_extra_vars('mobilestore_count',str(ct),'')
|
|
|
44 |
flag = da.get_extra_vars('mobilestore_flag')
|
|
|
45 |
|