WebSVN – SmartDukaan – /trunk/PyDTR/src/scripts/FofoSolr.py

from elixir import *
from shop2020.clients.InventoryClient import InventoryClient
from shop2020.model.v1.catalog.impl import DataService
from shop2020.model.v1.catalog.impl.DataService import Item, Tag_Listing, \
    Tag_Ranking, PrivateDeals
from shop2020.thriftpy.model.v1.catalog.ttypes import status
from sqlalchemy.sql.expression import or_, desc
from sqlalchemy.sql.functions import now
import json
import optparse
import os
import pymongo
import pysolr
import shutil
import traceback
from dtr.utils.utils import to_java_date
#import pymongo
#import pysolr


parser = optparse.OptionParser()
parser.add_option("-d", "--d", dest="dbHost",
                      default="127.0.0.1",
                      type="string", help="The HOST where the mysql server is running",
                      metavar="DBHOST")
parser.add_option("-s", "--s", dest="solrPath",
                      default="http://localhost:8984/solr/                1",
                      type="string", help="Complete solr path",
                      metavar="SOLRHOST")
parser.add_option("-m", "--m", dest="mongoHost",
                      default="localhost",
                      type="string", help="Complete solr path",
                      metavar="HOST")

(options, args) = parser.parse_args()

con=None

DataService.initialize(db_hostname=options.dbHost)
solr = pysolr.Solr(options.solrPath, timeout=10)

def get_mongo_connection(host='localhost', port=27017):
    global con
    if con is None:
        print "Establishing connection %s host and port %d" %(host,port)
        try:
            con = pymongo.MongoClient(host, port)
        except Exception, e:
            print e
            return None
    return con

class __SkuInfo:
    
    def __init__(self, id, ids, brand, model_name, category_id, subCategoryId,thumbnail, title, brand_synonyms, model_name_synonyms, source_product_names, \
                 category, subCategory):
        #Skubundle id
        self.id = id
        #_id of skus
        self.ids = ids
        self.brand = brand
        self.model_name = model_name
        self.category_id = category_id
        self.subCategoryId = subCategoryId
        self.thumbnail = thumbnail 
        self.title= title
        self.brand_synonyms = brand_synonyms
        self.model_name_synonyms = model_name_synonyms
        self.source_product_names = source_product_names
        self.category = category
        self.subCategory = subCategory
        
    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__, 
            sort_keys=True, indent=4)
        
class __Catalog:
    
    def __init__(self, id, rank, title, items):
        #Skubundle id
        self.id = id
        self.rank = rank
        self.title = title
        self._childDocuments_ = items
    def toJSON(self):
        return json.dumps(self, default=lambda o: o.__dict__)
        
class __Item:
    def __init__(self, id, color, tags):
        self.id = id
        self.color = color
        self._childDocuments_ = tags
        def toJSON(self):
            return json.dumps(self, default=lambda o: o.__dict__)
class __Tag:
    def __init__(self, id, sellingPrice, mop):
        self.id = id
        self.sellingPrice = sellingPrice
        self.mop = mop
        
        
#solr = pysolr.Solr(options.solrPath, timeout=10)


def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj
    

def populateTagItems():
    
    
    tagRankingList = session.query(Tag_Ranking).order_by(desc(Tag_Ranking.rankPoints)).all()
    rankingList = []
    featureMap = {}
    for tagRanking in tagRankingList:
        rankingList.append(tagRanking.catalogItemId)
        featureMap[tagRanking.catalogItemId]=tagRanking.feature
    catalogMap = {}
    #stmt = session.query(PrivateDeals).filter_by(isActive=1).filter(now().between(PrivateDeals.startDate, PrivateDeals.endDate)).subquery()
    #query = session.query(Item, privateDealAlias.dealPrice).outerjoin((privateDealAlias, Item.id==privateDealAlias.item_id)).filter(Item.status != status.PHASED_OUT)
    tuples = session.query(Tag_Listing, Item).join((Item, Item.id==Tag_Listing.item_id)).filter(or_(Item.status==status.ACTIVE, Item.status==status.PAUSED_BY_RISK, Item.status==status.PARTIALLY_ACTIVE))
    projection={'defaultImageUrl':1}
    for tag, item in tuples:
        if item.brand == "Live Demo":
            continue
        if not catalogMap.has_key(item.catalog_item_id):
            catalogObj = {}
            catalogObj['feature'] = ""
            catalogObj['title'] = " ".join(filter(None, [item.brand, item.model_name,  item.model_number]))
            catalogObj['brand'] = (item.brand)
            if item.brand == 'Huawei' or item.brand == 'Honor':
                catalogObj['brand'] = ('Huawei', 'Honor')
            catalogObj['identifier'] = item.catalog_item_id
            catalogObj['items'] = {}
            catalogObj['hot_deals'] = "False"
            filterMap = {"_id":item.catalog_item_id}
            #Dont include it catalog not available
            try:
                catalogObj['imageUrl'] = get_mongo_connection(options.mongoHost).CONTENT.siteContent.find_one(filterMap, projection)['defaultImageUrl']
                print catalogObj['imageUrl']
            except:
                try:
                    catalogObj['imageUrl'] = 'http://api.profittill.com/uploads/campaigns/' + str(item.catalog_item_id )+ '.jpg'
                except:
                    traceback.print_exc()
            try: 
                catalogObj['rank'] = rankingList.index(item.catalog_item_id)
            except:
                #A very big number
                traceback.print_exc()
                catalogObj['rank'] = 50000000
            if featureMap.has_key(item.catalog_item_id) and featureMap.get(item.catalog_item_id) is not None:
                    catalogObj['feature'] = featureMap.get(item.catalog_item_id)
            #Mobile and tablets are showcased in same category
            catalogObj['categoryId'] = 3 if item.category in [10006, 10009] else 6   
            catalogObj['create_timestamp'] = to_java_date(tag.create_timestamp)
            catalogMap[item.catalog_item_id] = catalogObj

        catalogObj = catalogMap.get(item.catalog_item_id)

        if tag.hot_deals:
            catalogObj['hot_deals'] = "True"
        
        if not  catalogObj['items'].has_key(item.id):
            catalogObj['items'][item.id] = {'color': item.color.replace("f_", ""), 'tagPricing':[]}
        
        itemMap = catalogObj['items'].get(item.id)
        itemMap['tagPricing'].append(tag)
        
    
    catalogObjs = []
    for catalogId, catalogMap in catalogMap.iteritems():
        itemsMap = catalogMap['items']
        itemObjs = []
        for itemId, itemMap in itemsMap.iteritems():
            tags = itemMap['tagPricing']
            for tag in tags:
                itemObj = {'id':('itemtag-%s-%s'%(itemId, tag.tag_id)), 'color_s':itemMap['color'],  'itemId_i': itemId, 'tagId_i':tag.tag_id, 
                           'mop_f': tag.mop, 'sellingPrice_f': tag.selling_price}
            itemObjs.append(itemObj)
        catalogObj = {'id':'catalog' + str(catalogId), 'rank_i':catalogMap['rank'], 'title_s': catalogMap['title'],'hot_deals_b':catalogMap['hot_deals'], 
                      '_childDocuments_':itemObjs, 
                      'catalogId_i':catalogId, 'imageUrl_s': catalogMap['imageUrl'], 'feature_s':catalogMap['feature'],
                      'brand_ss': catalogMap['brand'], 'create_s':catalogMap['create_timestamp'], 'categoryId_i':catalogMap['categoryId']}
        print catalogObj
        catalogObjs.append(catalogObj)
    solr.delete(q='*:*')
    solr.add(catalogObjs)
        
        
        
            
        
    #items = Item.query.filter(Item.risky==True).filter(or_(Item.status==status.ACTIVE)).all()
#    global con
#    if con is None:
#        print "Establishing connection %s host and port %d" %(host,port)
#        try:
#            con = pymongo.MongoClient(host, port)
#        except Exception, e:
#            print e
#            return None
#    return con

def pushData():
    #rankPoints = populateRankPoints()
    populateTagItems()
    #orderedMap = orderIt
    #convertToSolrDoc() 
    
if __name__=='__main__':
    pushData()
    #solr.delete(q='*:*')
    #solr.add([{'_childDocuments_': [{'itemId_i': 26564L, 'mop_f': 100.0, 'tagId_i': 2L, 'sellingPrice_f': 2600.0, 'color_s': '', 'id': 'itemtag-26564-2'}, {'itemId_i': 1, 'mop_f': 460.0, 'tagId_i': 2, 'sellingPrice_f': 500.0, 'color_s': 'Black', 'id': 'dummy'}], 'catalogId_i': 1019937L, 'rank_i': 500000, 'id': 'catalog1019937', 'title_s': 'Micromax Spark Vdeo Q415'}, {'_childDocuments_': [{'itemId_i': 26691L, 'mop_f': 4500.0, 'tagId_i': 4L, 'sellingPrice_f': 4299.0, 'color_s': 'Gold', 'id': 'itemtag-26691-4'}], 'catalogId_i': 1020034L, 'rank_i': 500000, 'id': 'catalog1020034', 'title_s': 'Samsung Z2'}, {'_childDocuments_': [{'itemId_i': 21585L, 'mop_f': 33000.0, 'tagId_i': 4L, 'sellingPrice_f': 31990.0, 'color_s': '', 'id': 'itemtag-21585-4'}], 'catalogId_i': 1015720L, 'rank_i': 500000, 'id': 'catalog1015720', 'title_s': 'HTC One A9 16GB'}, {'_childDocuments_': [{'itemId_i': 15663L, 'mop_f': 8000.0, 'tagId_i': 4L, 'sellingPrice_f': 7529.0, 'color_s': 'Black', 'id': 'itemtag-15663-4'}], 'catalogId_i': 1010735L, 'rank_i': 500000, 'id': 'catalog1010735', 'title_s': 'HTC Desire 210 Dual SIM'}, {'_childDocuments_': [{'itemId_i': 15549L, 'mop_f': 11000.0, 'tagId_i': 4L, 'sellingPrice_f': 10599.0, 'color_s': 'Matte Blue', 'id': 'itemtag-15549-4'}], 'catalogId_i': 1010312L, 'rank_i': 500000, 'id': 'catalog1010312', 'title_s': 'HTC Desire 310'}, {'_childDocuments_': [{'itemId_i': 18083L, 'mop_f': 10500.0, 'tagId_i': 4L, 'sellingPrice_f': 9999.0, 'color_s': 'Black', 'id': 'itemtag-18083-4'}], 'catalogId_i': 1012434L, 'rank_i': 500000, 'id': 'catalog1012434', 'title_s': 'HTC Desire 326G'}, {'_childDocuments_': [{'itemId_i': 26864L, 'mop_f': 3500.0, 'tagId_i': 4L, 'sellingPrice_f': 3299.0, 'color_s': 'Black', 'id': 'itemtag-26864-4'}], 'catalogId_i': 1020180L, 'rank_i': 500000, 'id': 'catalog1020180', 'title_s': 'Intex Aqua A4'}, {'_childDocuments_': [{'itemId_i': 26872L, 'mop_f': 4500.0, 'tagId_i': 4L, 'sellingPrice_f': 4299.0, 'color_s': 'Black', 'id': 'itemtag-26872-4'}], 'catalogId_i': 1020185L, 'rank_i': 500000, 'id': 'catalog1020185', 'title_s': 'Karbonn Indian 9 4G VoLTE'}, {'_childDocuments_': [{'itemId_i': 16404L, 'mop_f': 13950.0, 'tagId_i': 4L, 'sellingPrice_f': 13799.0, 'color_s': 'Pearl White', 'id': 'itemtag-16404-4'}], 'catalogId_i': 1011256L, 'rank_i': 500000, 'id': 'catalog1011256', 'title_s': 'HTC Desire 516 Dual SIM'}, {'_childDocuments_': [{'itemId_i': 26860L, 'mop_f': 3300.0, 'tagId_i': 4L, 'sellingPrice_f': 3199.0, 'color_s': 'Black', 'id': 'itemtag-26860-4'}], 'catalogId_i': 1020025L, 'rank_i': 500000, 'id': 'catalog1020025', 'title_s': 'Karbonn A40 Indian'}, {'_childDocuments_': [{'itemId_i': 26614L, 'mop_f': 4000.0, 'tagId_i': 4L, 'sellingPrice_f': 3890.0, 'color_s': 'Champagne', 'id': 'itemtag-26614-4'}], 'catalogId_i': 1019962L, 'rank_i': 500000, 'id': 'catalog1019962', 'title_s': 'Intex Aqua Strong 5.1 Plus'}])
Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/scripts/FofoSolr.py – Rev 24859
