Subversion Repositories SmartDukaan

import os
import subprocess
import sys
from datastore.DataAccessor import DataHelper

"""
Documentation for script runinfibeam.py
This is the script called by consetup.py
First it will run the spider for infibeam dynamically (i.e for determining no of pages)
Then, it will generate the csv file
@param  path to the folder in which spider-projects reside (:/home/gaurav/code) but start with pathsep
"""

ps = os.pathsep
ds = os.sep
path =  sys.argv[1]
path0 = path[1:len(path)]+ds+"ScrapyDatastore"+ds+"src"
path1 = path+ds+"ExtraFunctions"+ds+"src"
path2 = path+ds+"filterforname"+ds+"src"
path3 = path0+path1+path2
os.environ['PYTHONPATH'] = path3

path3 = str(os.getenv("PYTHONPATH"))
path4 = path+ds+"infibeamScrapy"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4+ds+"scrapy-ctl.py"

da = DataHelper()
da.initxy()
#da = DataHelper()
ct = 14
da.set_extra_vars('infibeam_count',str(ct),'')
da.set_extra_vars('infibeam_fails','0','')
da.set_extra_vars('infibeam_flag','TRUE','')

flag = da.get_extra_vars('infibeam_flag')
while flag=='TRUE':
        subprocess.call(["python", path4, "crawl", "infibeam"])
        ct = ct+1
        da.set_extra_vars('infibeam_count',str(ct),'')
        flag = da.get_extra_vars('infibeam_flag')