Subversion Repositories SmartDukaan

import pysolr

import pymongo

import json

import os

import optparse

import shutil

con=None

filePath = '/tmp/autosuggest.json'

destFilePath = '/home/kshitij/'

categoryMap = {3:'Mobiles',5:'Tablets',6:'Accessories'}

SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}

xstr = lambda s: s or ""

parser = optparse.OptionParser()

parser.add_option("-m", "--m", dest="mongoHost",

                      default="localhost",

                      type="string", help="The HOST where the mongo server is running",

                      metavar="HOST")

parser.add_option("-s", "--s", dest="server",

                      default="localhost",

                      type="string", help="The HOST where the mongo server is running",

                      metavar="HOST")

(options, args) = parser.parse_args()

synonymsMap = {}

modelNameSynonymMap = {}

class __SkuInfo:

    def __init__(self, id, skuBundleId, brand, model_name, source_product_name, category_id, category, available_price, mrp, in_stock, \

                 source_id, store, title, thumbnail, priceUpdatedOn, brand_synonyms, model_name_synonyms):

        self.id = id

        self.skuBundleId = skuBundleId

        self.brand = brand

        self.model_name = model_name

        self.source_product_name = source_product_name

        self.category_id = category_id

        self.category = category

        self.available_price = available_price

        self.mrp = mrp

        self.in_stock = in_stock

        self.source_id = source_id

        self.store = store

        self.title= title

        self.thumbnail = thumbnail

        self.priceUpdatedOn = priceUpdatedOn

        self.brand_synonyms = brand_synonyms

        self.model_name_synonyms = model_name_synonyms

solr = pysolr.Solr("http://104.200.25.40:8080/solr/", timeout=10)

def pushData():

    populateBrandSynonymMap()

    populateModelNameSynonyms()

    l = []

    exception = []

    catalogMap = {}

    items = get_mongo_connection(options.mongoHost).Catalog.MasterData.find({'source_id' : { "$in": SOURCE_MAP.keys() } })

    print items.count()

    for item in items:

        try:

            title = xstr(item['brand'])+" "+xstr(item['model_name'])

        except:

            exception.append(item['_id'])

            continue

        if not catalogMap.has_key(str(item['skuBundleId'])+" "+title):

            catalogMap[str(item['skuBundleId'])+" "+title] = [title]

        try:

            s_info = __SkuInfo(str(item['_id']),int(item['skuBundleId']),(item['brand']),(item['model_name']),(item['source_product_name']),int(item['category_id']),categoryMap.get(item['category_id']),float(item['available_price']),float(item['mrp']),item['in_stock'], \

                        int(item['source_id']),(item['source']),title,(item['thumbnail']), long(item['priceUpdatedOn']), synonymsMap.get(item['brand'].upper()), modelNameSynonymMap.get(item['skuBundleId']))

        except Exception as e:

            print "Exception in ",str(item['_id'])

            import sys

            sys.exit(1)

        l.append(s_info.__dict__)

    solr.add(l,boost={'source_product_name': '2.0',})

    for x in exception:

        print x

    autoSuggestList = []

    for v in catalogMap.itervalues():

        autoSuggestList.append({'val':v[0]})

    PATH = os.path.expanduser(filePath)

    if os.path.isfile(PATH):

        try:

            os.remove(os.path.expanduser(filePath))

        except OSError:

            pass

    output = open(filePath, 'ab+')

    json.dump(autoSuggestList, output)

    output.close()

    if options.server !='localhost':

        try:

            os.system("scp "+filePath +" root@"+options.server+":"+destFilePath)

        except:

            shutil.copy2(filePath, destFilePath)

def populateBrandSynonymMap(): 

    global synonymsMap

    brand_synonyms = get_mongo_connection(options.mongoHost).Catalog.BrandSynonyms.find()

    for synonym in brand_synonyms:

        synonymsMap[synonym['brand'].upper()] = synonym['synonyms']

def populateModelNameSynonyms():

    global modelNameSynonymMap

    model_synonyms = get_mongo_connection(options.mongoHost).Catalog.ModelNameSynonyms.find()

    for synonym in model_synonyms:

        modelNameSynonymMap[synonym['skuBundleId']] = synonym['synonyms'] 

def get_mongo_connection(host='localhost', port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

if __name__=='__main__':

    pushData()