Subversion Repositories SmartDukaan

from datastore.DataAccessor import DataHelper
import os
import sys
import subprocess
from mail import *

"""
Documentation for script consetup.py
This is the first script called by test.py, this file is created because before importing
datasore, its PYTHONPATH must be set which is done in test.py, so now we can import it.
Task of this script is to run the spiders for individual suppliers and call the modules to
generate respective csv files.
In the code, for each supplier a separate section is made, in that section last part is for 
csv file generation, rest of the part is for spiders. 
initxy method is called so as to make database visible in this scope
@param  path to the folder in which spider-projects reside (:/home/gaurav/code) but start with pathsep
"""


path =  sys.argv[1]
ps = os.pathsep
ds = os.sep
path0 = path[1:len(path)]+ds+"ScrapyDatastore"+ds+"src"
path1 = path+ds+"ExtraFunctions"+ds+"src"
path2 = path+ds+"filterforname"+ds+"src"
path3 = path0+path1+path2
os.environ['PYTHONPATH'] = path3

da = DataHelper()
da.initxy()
da.add_newcrawler()
attach = ""

# infibeam
path4 = path
path4 = path4.replace(ps,"")
path4 = path4+ds+"runinfibeam.py"
subprocess.call(["python", path4, path])


phones = da.get_all_infibeam_data()
if not phones:
    mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for infibeam spider. This mail is automatically generated via python script",attach)

path3 = str(os.getenv("PYTHONPATH"))
path4 = path+ds+"data2csv"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4+ds+"data2csv"+ds+"testfilteredinfibeam.py"
subprocess.call(["python", path4])


#babuchak
path3 = str(os.getenv("PYTHONPATH"))
path4 = path+ds+"babuchakScrapypass1"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"scrapy-ctl.py"
subprocess.call(["python", path4, "crawl", "babuchak"])

urls = da.get_allbabuchakurls()
if not urls:
    mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for first babuchak spider. This mail is automatically generated via python script",attach)

path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"babuchakScrapypass2"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"scrapy-ctl.py"
subprocess.call(["python", path4, "crawl", "babuchak1"])

purls = da.get_allbabuchakphoneurls()
if urls and not purls:
    mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for second babuchak spider. This mail is automatically generated via python script",attach)

path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"babuchakScrapypass3"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"scrapy-ctl.py"
subprocess.call(["python", path4, "crawl", "babuchak2"])

phones = da.get_allbabuchakphones()
if urls and purls and not phones:
    mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for third babuchak spider. This mail is automatically generated via python script",attach)

path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"data2csv"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"data2csv"+ds+"testbabuchak.py"
subprocess.call(["python", path4])



#indiaplaza
path4 = path
path4 = path4.replace(ps,"")
path4 = path4+ds+"runindiaplaza.py"
subprocess.call(["python", path4, path])

purls = da.get_all_ipbasic()
if not purls:
    mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for first indiaplaza spider. This mail is automatically generated via python script",attach)

path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"indiaplazascapypass2"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"scrapy-ctl.py"
subprocess.call(["python", path4, "crawl", "indiaplaza1"])


phones = da.get_all_indiaplaza_phones()
if purls and not phones:
    mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for second indiaplaza spider. This mail is automatically generated via python script",attach)

path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"data2csv"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"data2csv"+ds+"testfilterindiaplaza.py"
subprocess.call(["python", path4])

#This section commented out by ashish as it was running in infinite loop
#naaptol
#path3 = str(os.getenv("PYTHONPATH"))
#path4 = path + ds+"naaptolpass1"+ds+"src"
#path3 = path3 + path4
#os.putenv("PYTHONPATH",path3)
#path4 = path4.replace(ps,"")
#path4 = path4 + ds+"scrapy-ctl.py"
#subprocess.call(["python", path4, "crawl", "naaptol"])

#urls = da.get_allnaaptolurls()
#if not urls:
#       mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for first naaptol spider. This mail is automatically generated via python script",attach)

#path3 = str(os.getenv("PYTHONPATH"))
#path4 = path + ds+"naaptolpass2"+ds+"src"
#path3 = path3 + path4
#os.putenv("PYTHONPATH",path3)
#path4 = path4.replace(ps,"")
#path4 = path4 + ds+"scrapy-ctl.py"
#subprocess.call(["python", path4, "crawl", "naaptol1"])

#path3 = str(os.getenv("PYTHONPATH"))
#path4 = path + ds+"naaptolpass3"+ds+"src"
#path3 = path3 + path4
#os.putenv("PYTHONPATH",path3)
#path4 = path4.replace(ps,"")
#path4 = path4 + ds+"scrapy-ctl.py"
#subprocess.call(["python", path4, "crawl", "naaptol2"])

#phones = da.get_allnaaptolphones()
#if urls and not phones:
#       mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for second or third naaptol spider. This mail is automatically generated via python script",attach)

#path3 = str(os.getenv("PYTHONPATH"))
#path4 = path + ds+"data2csv"+ds+"src"
#path3 = path3 + path4
#os.putenv("PYTHONPATH",path3)
#path4 = path4.replace(ps,"")
#path4 = path4 + ds+"data2csv"+ds+"testnaaptol.py"
#subprocess.call(["python", path4])


#univercell
path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"universalScrapypass1"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"scrapy-ctl.py"
subprocess.call(["python", path4, "crawl", "univercell"])

vendors = da.get_all_univervendors()
if not vendors:
        mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for first univercell spider. This mail is automatically generated via python script",attach)

path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"univercellScrapypass2"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"scrapy-ctl.py"
subprocess.call(["python", path4, "crawl", "univercell1"])

phones = da.get_all_univercell_phones()
if vendors and not phones:
        mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for second univercell spider. This mail is automatically generated via python script",attach)

path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"data2csv"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"data2csv"+ds+"testfilterunivercell.py"
subprocess.call(["python", path4])


#mobilestore
path4 = path
path4 = path4.replace(ps,"")
path4 = path4+ds+"runmobstore.py"
subprocess.call(["python", path4, path])

phones = da.get_allmobstorephones_new()
if not phones:
        mail("ashish.singh@shop2020.in","Bug","There is some mistake in parameters for mobilestore spider. This mail is automatically generated via python script",attach)


path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"data2csv"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"data2csv"+ds+"new_mobilestore.py"
subprocess.call(["python", path4])


#call indexing
path3 = str(os.getenv("PYTHONPATH"))
path4 = path + ds+"Lucene"+ds+"src"
path3 = path3 + path4
os.putenv("PYTHONPATH",path3)
path4 = path4.replace(ps,"")
path4 = path4 + ds+"Searching"+ds+"Searchbymodel.py"
subprocess.call(["python", path4])