Subversion Repositories SmartDukaan

Rev

Rev 18252 | Rev 20286 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 18252 Rev 20285
Line 3... Line 3...
3 
import json
 3 
import json
4 
import os
 4 
import os
5 
import optparse
 5 
import optparse
6 
import shutil
 6 
import shutil
7 
import traceback
 7 
import traceback
- 
 
 8 
from dtr.utils.utils import CATEGORY_MAP, SUB_CATEGORY_MAP
8 
 
 9 
 
9 
con=None
 10 
con=None
10 
 
 11 
 
11 
filePath = '/tmp/autosuggest.json'
 - 
 
12 
destFilePath = '/home/kshitij/'
 - 
 
13 
 
 - 
 
14 
categoryMap = {3:'Mobiles',5:'Tablets',6:'Accessories'}
 - 
 
15 
SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}
 12 
SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}
16 
xstr = lambda s: s or ""
 13 
xstr = lambda s: s or ""
17 
 
 14 
 
18 
parser = optparse.OptionParser()
 15 
parser = optparse.OptionParser()
19 
parser.add_option("-m", "--m", dest="mongoHost",
 16 
parser.add_option("-m", "--m", dest="mongoHost",
20 
                      default="localhost",
 17 
                      default="localhost",
21 
                      type="string", help="The HOST where the mongo server is running",
 18 
                      type="string", help="The HOST where the mongo server is running",
22 
                      metavar="HOST")
 19 
                      metavar="HOST")
23 
parser.add_option("-s", "--s", dest="server",
 20 
parser.add_option("-s", "--s", dest="solrPath",
24 
                      default="localhost",
 21 
                      default="http://localhost:8080/solr/",
25 
                      type="string", help="The HOST where the mongo server is running",
 22 
                      type="string", help="Complete solr path",
26 
                      metavar="HOST")
 23 
                      metavar="HOST")
27 
 
 24 
 
28 
(options, args) = parser.parse_args()
 25 
(options, args) = parser.parse_args()
29 
 
 26 
 
30 
synonymsMap = {}
 27 
synonymsMap = {}
31 
modelNameSynonymMap = {}
 28 
modelNameSynonymMap = {}
32 
 
 29 
 
33 
 
 30 
 
34 
class __SkuInfo:
 31 
class __SkuInfo:
35
32
36 
    def __init__(self, id, skuBundleId, brand, model_name, source_product_name, category_id, category, available_price, mrp, in_stock, \
 33 
    def __init__(self, id, ids, brand, model_name, category_id, subCategoryId,thumbnail, title, brand_synonyms, model_name_synonyms, source_product_names, \
37 
                 source_id, store, title, thumbnail, priceUpdatedOn, brand_synonyms, model_name_synonyms):
 34 
                 category, subCategory):
38 
        self.id = id
 35 
        self.id = id
39 
        self.skuBundleId = skuBundleId
 36 
        self.ids = ids
40 
        self.brand = brand
 37 
        self.brand = brand
41 
        self.model_name = model_name
 38 
        self.model_name = model_name
42 
        self.source_product_name = source_product_name
 - 
 
43 
        self.category_id = category_id
 39 
        self.category_id = category_id
44 
        self.category = category
 40 
        self.subCategoryId = subCategoryId
45 
        self.available_price = available_price
 - 
 
46 
        self.mrp = mrp
 - 
 
47 
        self.in_stock = in_stock
 - 
 
48 
        self.source_id = source_id
 41 
        self.thumbnail = thumbnail
49 
        self.store = store
 - 
 
50 
        self.title= title
 42 
        self.title= title
51 
        self.thumbnail = thumbnail
 - 
 
52 
        self.priceUpdatedOn = priceUpdatedOn
 - 
 
53 
        self.brand_synonyms = brand_synonyms
 43 
        self.brand_synonyms = brand_synonyms
54 
        self.model_name_synonyms = model_name_synonyms
 44 
        self.model_name_synonyms = model_name_synonyms
- 
 
 45 
        self.source_product_names = source_product_names
- 
 
 46 
        self.category = category
- 
 
 47 
        self.subCategory = subCategory
55
48
56
49
57 
solr = pysolr.Solr("http://104.200.25.40:8080/solr/", timeout=10)
 50 
solr = pysolr.Solr(options.solrPath, timeout=10)
58 
 
 51 
 
59 
def pushData():
 52 
def pushData():
60 
    populateBrandSynonymMap()
 53 
    populateBrandSynonymMap()
61 
    populateModelNameSynonyms()
 54 
    populateModelNameSynonyms()
62 
    l = []
 55 
    l = []
Line 68... Line 61...
68 
        try:
 61 
        try:
69 
            title = xstr(item['brand'])+" "+xstr(item['model_name'])
 62 
            title = xstr(item['brand'])+" "+xstr(item['model_name'])
70 
        except:
 63 
        except:
71 
            exception.append(item['_id'])
 64 
            exception.append(item['_id'])
72 
            continue
 65 
            continue
73 
        if not catalogMap.has_key(str(item['skuBundleId'])+" "+title):
 - 
 
74 
            catalogMap[str(item['skuBundleId'])+" "+title] = [title]
 - 
 
75 
        try:
 - 
 
76 
            s_info = __SkuInfo(str(item['_id']),int(item['skuBundleId']),(item['brand']),(item['model_name']),(item['source_product_name']),int(item['category_id']),categoryMap.get(item['category_id']),float(item['available_price']),float(item['mrp']),item['in_stock'], \
 - 
 
77 
                        int(item['source_id']),(item['source']),title,(item['thumbnail']), long(item['priceUpdatedOn']), synonymsMap.get(item['brand'].upper()), modelNameSynonymMap.get(item['skuBundleId']))
 - 
 
78 
        except:
 - 
 
79 
            traceback.print_exc()
 - 
 
80 
            continue
 - 
 
81 
        l.append(s_info.__dict__)
 - 
 
82
66
- 
 
 67 
        if catalogMap.has_key(int(item['skuBundleId'])):
- 
 
 68 
            skuInfo = catalogMap.get(int(item['skuBundleId']))
83 
    solr.add(l,boost={'source_product_name': '2.0',})
 69 
            skuInfo.source_product_names.append(item['source_product_name'].strip())
- 
 
 70 
            skuInfo.ids.append(int(item['_id']))
- 
 
 71 
        else:
- 
 
 72 
            syn_brand = synonymsMap.get((item['brand']).upper())
- 
 
 73 
            model_syn = modelNameSynonymMap.get(item['skuBundleId'])
- 
 
 74 
            if len(str(item['thumbnail']).strip()) == 0:
- 
 
 75 
                item['thumbnail'] = "http://api.profittill.com/img/no_image_available.png"
- 
 
 76
- 
 
 77 
            skuInfo = __SkuInfo(str(item['skuBundleId']), [int(item['_id'])], str(item['brand']).strip(), str(item['model_name']).strip(), \
- 
 
 78 
                                int(item['category_id']), int(item['subCategoryId']), item['thumbnail'].strip(), title, syn_brand, model_syn, [item['source_product_name'].strip()], \
- 
 
 79 
                                CATEGORY_MAP.get(int(item['category_id'])), SUB_CATEGORY_MAP.get(int(item['subCategoryId'])))
- 
 
 80
- 
 
 81 
            catalogMap[int(item['skuBundleId'])] = skuInfo
- 
 
 82
- 
 
 83 
    for data in catalogMap.values():
- 
 
 84 
        l.append(data.__dict__)
- 
 
 85 
    solr.add(l)
84
86
85
87
86 
    for x in exception:
 88 
    for x in exception:
87 
        print x
 89 
        print x
88
- 
 
89 
    autoSuggestList = []
 - 
 
90 
    for v in catalogMap.itervalues():
 - 
 
91 
        autoSuggestList.append({'val':v[0]})
 - 
 
92 
    PATH = os.path.expanduser(filePath)
 - 
 
93 
    if os.path.isfile(PATH):
 - 
 
94 
        try:
 - 
 
95 
            os.remove(os.path.expanduser(filePath))
 - 
 
96 
        except OSError:
 - 
 
97 
            pass
 - 
 
98 
    output = open(filePath, 'ab+')
 - 
 
99 
    json.dump(autoSuggestList, output)
 - 
 
100 
    output.close()
 - 
 
101 
    if options.server !='localhost':
 - 
 
102 
        try:
 - 
 
103 
            os.system("scp "+filePath +" root@"+options.server+":"+destFilePath)
 - 
 
104 
        except:
 - 
 
105 
            shutil.copy2(filePath, destFilePath)
 - 
 
106
- 
 
107
- 
 
108 
 
 90 
 
109 
def populateBrandSynonymMap():
91 
def populateBrandSynonymMap():
110 
    global synonymsMap
 92 
    global synonymsMap
111 
    brand_synonyms = get_mongo_connection(options.mongoHost).Catalog.BrandSynonyms.find()
 93 
    brand_synonyms = get_mongo_connection(options.mongoHost).Catalog.BrandSynonyms.find()
112 
    for synonym in brand_synonyms:
 94 
    for synonym in brand_synonyms: