Subversion Repositories SmartDukaan

'''
Created on 30-Aug-2011

@author: Varun Gupta
'''
from lucene import File, StandardAnalyzer, IndexWriter, Version, SimpleFSDirectory, Document, Field
from PhonePriceDoc import PhonePriceDoc
from ScrapedDataManager import ScrapedDataManager
import lucene, Utils, json

class IndexBuilder:

    def __init__(self, price_data, new_index = True):
        self.indexDir = "/usr/price-comp-dashboard/lucene-index-dir"
        lucene.initVM()
        dir = SimpleFSDirectory(File(self.indexDir))
        self.analyzer = StandardAnalyzer(Version.LUCENE_30)
        self.writer = IndexWriter(dir, self.analyzer, new_index, IndexWriter.MaxFieldLength(512))
        self.price_data = price_data
        self.brand_model_extracter = Utils.BrandAndModelExtracter()
    
    def build(self):
        print "Currently there are %d documents in the index..." % self.writer.numDocs()
        count = 0
        
        for phone_price in self.price_data:
            print phone_price
            #doc = PhonePriceDoc(phone_price)
            
            brand, name = self.brand_model_extracter.extract(phone_price['name'])
            print "Brand: %s, Model: %s" % (brand, name)
            try:
                doc = Document()
                doc.add(Field("name", name, Field.Store.YES, Field.Index.ANALYZED))
                doc.add(Field("brand", brand, Field.Store.YES, Field.Index.ANALYZED))
                doc.add(Field("source", str(phone_price['source']), Field.Store.YES, Field.Index.ANALYZED))
                doc.add(Field("price", str(phone_price['price']), Field.Store.YES, Field.Index.NO))
                doc.add(Field("in_stock", str(phone_price['in_stock']), Field.Store.YES, Field.Index.NO))
                doc.add(Field("url", str(phone_price['product_url']), Field.Store.YES, Field.Index.NO))
                self.writer.addDocument(doc)
                count += 1
            except KeyError as e:
                print 'KeyError:', e
                print phone_price
            
        print "Indexed lines from stdin (%d documents in index)" % (self.writer.numDocs())
        print "About to optimize index of %d documents..." % self.writer.numDocs()
        self.writer.optimize()
        print "...done optimizing index of %d documents" % self.writer.numDocs()
        print "Closing index of %d documents..." % self.writer.numDocs()
        print "%d docs added" % count
        self.writer.close()
        
if __name__ == '__main__':
    data_manager = ScrapedDataManager()
    fp = open(data_manager.price_data_file, 'r')
    phones = json.load(fp)
    print phones.__len__()
    indexer = IndexBuilder(price_data = phones, new_index = True)
    indexer.build()
#    catalog_client = InventoryClient().get_client()
#    items = catalog_client.getAllItems(True)
#    print phones
#    for item in items:  print item.id, item.brand, item.modelName, item.modelNumber