Subversion Repositories SmartDukaan

Rev

Rev 20288 | Rev 20291 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13828 kshitij.so 1 
import pysolr
14186 kshitij.so 2 
import pymongo
13828 kshitij.so 3 
import json
14186 kshitij.so 4 
import os
14243 kshitij.so 5 
import optparse
14245 kshitij.so 6 
import shutil
18252 kshitij.so 7 
import traceback
20285 kshitij.so 8 
from dtr.utils.utils import CATEGORY_MAP, SUB_CATEGORY_MAP
13828 kshitij.so 9 
 
10 
con=None
11 
 
17790 kshitij.so 12 
SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}
13828 kshitij.so 13 
xstr = lambda s: s or ""
14 
 
14243 kshitij.so 15 
parser = optparse.OptionParser()
16 
parser.add_option("-m", "--m", dest="mongoHost",
17 
                      default="localhost",
18 
                      type="string", help="The HOST where the mongo server is running",
19 
                      metavar="HOST")
20285 kshitij.so 20 
parser.add_option("-s", "--s", dest="solrPath",
21 
                      default="http://localhost:8080/solr/",
22 
                      type="string", help="Complete solr path",
14248 kshitij.so 23 
                      metavar="HOST")
14243 kshitij.so 24 
 
25 
(options, args) = parser.parse_args()
26 
 
15067 kshitij.so 27 
synonymsMap = {}
28 
modelNameSynonymMap = {}
14243 kshitij.so 29 
 
15067 kshitij.so 30 
 
13828 kshitij.so 31 
class __SkuInfo:
32 
 
20285 kshitij.so 33 
    def __init__(self, id, ids, brand, model_name, category_id, subCategoryId,thumbnail, title, brand_synonyms, model_name_synonyms, source_product_names, \
34 
                 category, subCategory):
13828 kshitij.so 35 
        self.id = id
20285 kshitij.so 36 
        self.ids = ids
13828 kshitij.so 37 
        self.brand = brand
38 
        self.model_name = model_name
39 
        self.category_id = category_id
20285 kshitij.so 40 
        self.subCategoryId = subCategoryId
41 
        self.thumbnail = thumbnail
13828 kshitij.so 42 
        self.title= title
15067 kshitij.so 43 
        self.brand_synonyms = brand_synonyms
44 
        self.model_name_synonyms = model_name_synonyms
20285 kshitij.so 45 
        self.source_product_names = source_product_names
46 
        self.category = category
47 
        self.subCategory = subCategory
13828 kshitij.so 48 
 
49 
 
20285 kshitij.so 50 
solr = pysolr.Solr(options.solrPath, timeout=10)
13828 kshitij.so 51 
 
52 
def pushData():
15067 kshitij.so 53 
    populateBrandSynonymMap()
15068 kshitij.so 54 
    populateModelNameSynonyms()
13828 kshitij.so 55 
    l = []
17790 kshitij.so 56 
    exception = []
14186 kshitij.so 57 
    catalogMap = {}
17791 kshitij.so 58 
    items = get_mongo_connection(options.mongoHost).Catalog.MasterData.find({'source_id' : { "$in": SOURCE_MAP.keys() } })
13828 kshitij.so 59 
    print items.count()
60 
    for item in items:
17790 kshitij.so 61 
        try:
62 
            title = xstr(item['brand'])+" "+xstr(item['model_name'])
63 
        except:
64 
            exception.append(item['_id'])
65 
            continue
17996 kshitij.so 66 
 
20285 kshitij.so 67 
        if catalogMap.has_key(int(item['skuBundleId'])):
68 
            skuInfo = catalogMap.get(int(item['skuBundleId']))
20289 kshitij.so 69 
            print item['_id']
70 
            print item['source_product_name']
20285 kshitij.so 71 
            skuInfo.source_product_names.append(item['source_product_name'].strip())
72 
            skuInfo.ids.append(int(item['_id']))
73 
        else:
74 
            syn_brand = synonymsMap.get((item['brand']).upper())
75 
            model_syn = modelNameSynonymMap.get(item['skuBundleId'])
20286 kshitij.so 76 
            try:
20287 kshitij.so 77 
                if len(str(item['thumbnail']).strip()) == 0:
78 
                    item['thumbnail'] = "http://api.profittill.com/img/no_image_available.png"
79 
            except:
80 
                print "Exception in item thumbnail",item['_id']
20288 kshitij.so 81 
                continue
20287 kshitij.so 82 
            try:
20286 kshitij.so 83 
                skuInfo = __SkuInfo(str(item['skuBundleId']), [int(item['_id'])], str(item['brand']).strip(), str(item['model_name']).strip(), \
20285 kshitij.so 84 
                                int(item['category_id']), int(item['subCategoryId']), item['thumbnail'].strip(), title, syn_brand, model_syn, [item['source_product_name'].strip()], \
85 
                                CATEGORY_MAP.get(int(item['category_id'])), SUB_CATEGORY_MAP.get(int(item['subCategoryId'])))
20286 kshitij.so 86 
            except:
87 
                print "Exception in item ",item['_id']
20288 kshitij.so 88 
                continue
20285 kshitij.so 89 
            catalogMap[int(item['skuBundleId'])] = skuInfo
90 
 
91 
    for data in catalogMap.values():
92 
        l.append(data.__dict__)
93 
    solr.add(l)
14186 kshitij.so 94 
 
17790 kshitij.so 95 
 
96 
    for x in exception:
97 
        print x
13828 kshitij.so 98 
 
15067 kshitij.so 99 
def populateBrandSynonymMap():
100 
    global synonymsMap
101 
    brand_synonyms = get_mongo_connection(options.mongoHost).Catalog.BrandSynonyms.find()
102 
    for synonym in brand_synonyms:
103 
        synonymsMap[synonym['brand'].upper()] = synonym['synonyms']
104 
 
105 
def populateModelNameSynonyms():
106 
    global modelNameSynonymMap
107 
    model_synonyms = get_mongo_connection(options.mongoHost).Catalog.ModelNameSynonyms.find()
108 
    for synonym in model_synonyms:
109 
        modelNameSynonymMap[synonym['skuBundleId']] = synonym['synonyms']
110 
 
13828 kshitij.so 111 
def get_mongo_connection(host='localhost', port=27017):
112 
    global con
113 
    if con is None:
114 
        print "Establishing connection %s host and port %d" %(host,port)
115 
        try:
116 
            con = pymongo.MongoClient(host, port)
117 
        except Exception, e:
118 
            print e
119 
            return None
120 
    return con
14186 kshitij.so 121 
 
13828 kshitij.so 122 
if __name__=='__main__':
17790 kshitij.so 123 
    pushData()