Subversion Repositories SmartDukaan

Rev

Rev 4198 | Rev 5507 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 4198 Rev 5291
Line 1... Line 1...
1 
'''
 1 
'''
2 
Created on 16-Sep-2011
 2 
Created on 16-Sep-2011
3 
 
 3 
 
4 
@author: Varun Gupta
 4 
@author: Varun Gupta
5 
'''
 5 
'''
6 
import sys
 6 
import sys, json
7 
from Clients.GAEServletClient import url, clearPriceData, initJobQueue, getPhonePricesJSON
 7 
from Clients.GAEServletClient import url, clearPriceData, initJobQueue, getPhonePricesJSON
8 
from ScraperAgent import ScraperAgent
 8 
from ScraperAgent import ScraperAgent
9 
from PyLucene.IndexBuilder import IndexBuilder
 9 
from PyLucene.IndexBuilder import IndexBuilder
10 
 
 10 
 
11 
def startScraper():
 11 
def startScraper():
12 
    ScraperAgent().start()
 12 
    ScraperAgent().start()
13 
 
 13 
 
14 
def buildIndex():
 14 
def buildIndex():
15 
    price_data = getPhonePricesJSON(url)
 15 
    #price_data = getPhonePricesJSON(url)
- 
 
 16 
    f = open('/tmp/price-comp-dashboard/primary-crawl.json')
- 
 
 17 
    price_data = json.load(f)
- 
 
 18 
    print len(price_data)
16 
    indexer = IndexBuilder(price_data = price_data, new_index = False)
 19 
    indexer = IndexBuilder(price_data = price_data, new_index = True)
17 
    indexer.build()
 20 
    indexer.build()
18 
 
 21 
 
19 
if __name__ == '__main__':
 22 
if __name__ == '__main__':
20 
    try:
 23 
    try:
21 
        cmd = sys.argv[1].strip()
 24 
        cmd = sys.argv[1].strip()