Subversion Repositories SmartDukaan

Rev

Rev 20663 | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13828 kshitij.so 1
import pysolr
14186 kshitij.so 2
import pymongo
13828 kshitij.so 3
import json
14186 kshitij.so 4
import os
14243 kshitij.so 5
import optparse
14245 kshitij.so 6
import shutil
18252 kshitij.so 7
import traceback
20285 kshitij.so 8
from dtr.utils.utils import CATEGORY_MAP, SUB_CATEGORY_MAP
13828 kshitij.so 9
 
10
con=None
11
 
17790 kshitij.so 12
SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}
13828 kshitij.so 13
xstr = lambda s: s or ""
14
 
14243 kshitij.so 15
parser = optparse.OptionParser()
16
parser.add_option("-m", "--m", dest="mongoHost",
17
                      default="localhost",
18
                      type="string", help="The HOST where the mongo server is running",
19
                      metavar="HOST")
20285 kshitij.so 20
parser.add_option("-s", "--s", dest="solrPath",
20322 kshitij.so 21
                      default="http://localhost:8983/solr/",
20285 kshitij.so 22
                      type="string", help="Complete solr path",
14248 kshitij.so 23
                      metavar="HOST")
14243 kshitij.so 24
 
25
(options, args) = parser.parse_args()
26
 
15067 kshitij.so 27
synonymsMap = {}
28
modelNameSynonymMap = {}
14243 kshitij.so 29
 
15067 kshitij.so 30
 
13828 kshitij.so 31
class __SkuInfo:
32
 
20285 kshitij.so 33
    def __init__(self, id, ids, brand, model_name, category_id, subCategoryId,thumbnail, title, brand_synonyms, model_name_synonyms, source_product_names, \
34
                 category, subCategory):
22240 amit.gupta 35
        #Skubundle id
13828 kshitij.so 36
        self.id = id
22240 amit.gupta 37
        #_id of skus
20285 kshitij.so 38
        self.ids = ids
13828 kshitij.so 39
        self.brand = brand
40
        self.model_name = model_name
41
        self.category_id = category_id
20285 kshitij.so 42
        self.subCategoryId = subCategoryId
43
        self.thumbnail = thumbnail 
13828 kshitij.so 44
        self.title= title
15067 kshitij.so 45
        self.brand_synonyms = brand_synonyms
46
        self.model_name_synonyms = model_name_synonyms
20285 kshitij.so 47
        self.source_product_names = source_product_names
48
        self.category = category
49
        self.subCategory = subCategory 
13828 kshitij.so 50
 
51
 
20285 kshitij.so 52
solr = pysolr.Solr(options.solrPath, timeout=10)
13828 kshitij.so 53
 
54
def pushData():
15067 kshitij.so 55
    populateBrandSynonymMap()
15068 kshitij.so 56
    populateModelNameSynonyms()
17790 kshitij.so 57
    exception = []
14186 kshitij.so 58
    catalogMap = {}
20663 kshitij.so 59
    items = get_mongo_connection(options.mongoHost).Catalog.MasterData.find({'source_id' : { "$in": SOURCE_MAP.keys() },'category_id':{"$in":[3,5]} })
13828 kshitij.so 60
    print items.count()
61
    for item in items:
17790 kshitij.so 62
        try:
63
            title = xstr(item['brand'])+" "+xstr(item['model_name'])
64
        except:
65
            exception.append(item['_id'])
66
            continue
17996 kshitij.so 67
 
20285 kshitij.so 68
        if catalogMap.has_key(int(item['skuBundleId'])):
69
            skuInfo = catalogMap.get(int(item['skuBundleId']))
70
            skuInfo.source_product_names.append(item['source_product_name'].strip())
71
            skuInfo.ids.append(int(item['_id']))
72
        else:
73
            syn_brand = synonymsMap.get((item['brand']).upper())
74
            model_syn = modelNameSynonymMap.get(item['skuBundleId'])
20286 kshitij.so 75
            try:
20287 kshitij.so 76
                if len(str(item['thumbnail']).strip()) == 0:
77
                    item['thumbnail'] = "http://api.profittill.com/img/no_image_available.png"
78
            except:
79
                print "Exception in item thumbnail",item['_id']
20288 kshitij.so 80
                continue
20287 kshitij.so 81
            try:
20286 kshitij.so 82
                skuInfo = __SkuInfo(str(item['skuBundleId']), [int(item['_id'])], str(item['brand']).strip(), str(item['model_name']).strip(), \
20285 kshitij.so 83
                                int(item['category_id']), int(item['subCategoryId']), item['thumbnail'].strip(), title, syn_brand, model_syn, [item['source_product_name'].strip()], \
84
                                CATEGORY_MAP.get(int(item['category_id'])), SUB_CATEGORY_MAP.get(int(item['subCategoryId'])))
20286 kshitij.so 85
            except:
86
                print "Exception in item ",item['_id']
20288 kshitij.so 87
                continue
20285 kshitij.so 88
            catalogMap[int(item['skuBundleId'])] = skuInfo
20322 kshitij.so 89
    l=[]        
20285 kshitij.so 90
    for data in catalogMap.values():
91
        l.append(data.__dict__)
20322 kshitij.so 92
    solr.add(l)
93
 
17790 kshitij.so 94
    for x in exception:
95
        print x
13828 kshitij.so 96
 
15067 kshitij.so 97
def populateBrandSynonymMap(): 
98
    global synonymsMap
99
    brand_synonyms = get_mongo_connection(options.mongoHost).Catalog.BrandSynonyms.find()
100
    for synonym in brand_synonyms:
101
        synonymsMap[synonym['brand'].upper()] = synonym['synonyms']
102
 
103
def populateModelNameSynonyms():
104
    global modelNameSynonymMap
105
    model_synonyms = get_mongo_connection(options.mongoHost).Catalog.ModelNameSynonyms.find()
106
    for synonym in model_synonyms:
107
        modelNameSynonymMap[synonym['skuBundleId']] = synonym['synonyms'] 
108
 
13828 kshitij.so 109
def get_mongo_connection(host='localhost', port=27017):
110
    global con
111
    if con is None:
112
        print "Establishing connection %s host and port %d" %(host,port)
113
        try:
114
            con = pymongo.MongoClient(host, port)
115
        except Exception, e:
116
            print e
117
            return None
118
    return con
14186 kshitij.so 119
 
13828 kshitij.so 120
if __name__=='__main__':
17790 kshitij.so 121
    pushData()