Subversion Repositories SmartDukaan

from bson.objectid import ObjectId

from datetime import datetime, timedelta

from dtr.config import PythonPropertyReader

from dtr.dao import FeaturedDeals, AppTransactions, AppOfferObj

from dtr.storage import DataService

from dtr.storage.DataService import price_preferences, brand_preferences, \

    user_actions, Brands, app_offers, approved_app_transactions, user_app_cashbacks, \

    user_app_installs, appmasters, notification_campaigns

from dtr.storage.MemCache import MemCache

from dtr.utils.MailSender import Email

from dtr.utils.utils import to_java_date, CB_PENDING, CB_APPROVED, CB_INIT, \

    to_py_date, CB_REJECTED, SUB_CATEGORY_MAP

from sqlalchemy.sql.expression import func, func, or_, desc, asc, case

from elixir import *

from operator import itemgetter

from pymongo.command_cursor import CommandCursor

import pymongo

import random

import re

import time

import traceback

from itertools import groupby

import urllib

import urllib2

import json

import collections

try:

    import ordereddict

except:

    pass

con = None

es = None

from elasticsearch import Elasticsearch

from shop2020.config.client.ConfigClient import ConfigClient

DataService.initialize(db_hostname="localhost")

mc = MemCache("127.0.0.1")

config_client = ConfigClient()

elastic_search_host = config_client.get_property('elastic_search_host')

elastic_search_port = config_client.get_property('elastic_search_port')

SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}

COLLECTION_MAP = {

                  'ExceptionalNlc':'skuBundleId',

                  'SkuDealerPrices':'skuBundleId',

                  'SkuDiscountInfo':'skuBundleId',

                  'SkuSchemeDetails':'skuBundleId',

                  'DealPoints':'skuBundleId',

                  'FeaturedDeals': 'skuBundleId'

                  }

def get_elastic_search_connection():

    global es

    if es is None:

        print "Establishing connection with elastic search %s host and port %s"%(elastic_search_host,elastic_search_port)

        es = Elasticsearch([{'host': elastic_search_host, 'port': elastic_search_port}])

        return es

    return es

def get_mongo_connection(host='localhost', port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def populateCashBack():

    print "Populating cashback"

    cashBackMap = {}

    itemCashBackMap = {}

    cashBack = list(get_mongo_connection().Catalog.CategoryCashBack.find())

    for row in cashBack:

        temp_map = {}

        temp_list = []

        if cashBackMap.has_key(row['source_id']):

            arr = cashBackMap.get(row['source_id'])

            for val in arr:

                temp_list.append(val)

            temp_map[row['category_id']] = row

            temp_list.append(temp_map)

            cashBackMap[row['source_id']] = temp_list 

        else:

            temp_map[row['category_id']] = row

            temp_list.append(temp_map)

            cashBackMap[row['source_id']] = temp_list

    itemCashBack = list(get_mongo_connection().Catalog.ItemCashBack.find())

    for row in itemCashBack:

        if not itemCashBackMap.has_key(row['skuId']):

            itemCashBackMap[row['skuId']] = row

    mc.set("item_cash_back", itemCashBackMap, 24 * 60 * 60)

    mc.set("category_cash_back", cashBackMap, 24 * 60 * 60)

def addCategoryDiscount(data):

    collection = get_mongo_connection().Catalog.CategoryDiscount

    query = []

    data['brand'] = data['brand'].strip().upper()

    data['discountType'] = data['discountType'].upper().strip()

    query.append({"brand":data['brand']})

    query.append({"category_id":data['category_id']})

    r = collection.find({"$and":query})

    if r.count() > 0:

        return {0:"Brand & Category info already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'brand':data['brand'],'category_id':data['category_id']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def updateCategoryDiscount(data,_id):

    try:

        collection = get_mongo_connection().Catalog.CategoryDiscount

        collection.update({'_id':ObjectId(_id)},{"$set":{'min_discount':data['min_discount'],'max_discount':data['max_discount'],'discountType':data['discountType'].upper().strip()}},upsert=False, multi = False)

        get_mongo_connection().Catalog.MasterData.update({'brand':data['brand'],'category_id':data['category_id']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data updated successfully"}

    except:

        return {0:"Data not updated."}

def getAllCategoryDiscount():

    data = []

    collection = get_mongo_connection().Catalog.CategoryDiscount

    cursor = collection.find()

    for val in cursor:

        data.append(val)

    return data

def __getBundledSkusfromSku(sku):

    masterData =  get_mongo_connection().Catalog.MasterData.find_one({"_id":sku},{"skuBundleId":1})

    if masterData is not None:

        return list(get_mongo_connection().Catalog.MasterData.find({"skuBundleId":masterData.get('skuBundleId')},{'_id':1,'skuBundleId':1}))

    else:

        return []

def addSchemeDetailsForSku(data):

    collection = get_mongo_connection().Catalog.SkuSchemeDetails

    result = collection.find_one({'skuBundleId':data['skuBundleId']})

    if result is None:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

    return {1:"BundleId info already present"}

def getAllSkuWiseSchemeDetails(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.SkuSchemeDetails

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def addSkuDiscountInfo(data):

    collection = get_mongo_connection().Catalog.SkuDiscountInfo

    cursor = collection.find_one({"skuBundleId":data['skuBundleId']})

    if cursor is not None:

        return {0:"BundleId information already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def getallSkuDiscountInfo(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.SkuDiscountInfo

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def updateSkuDiscount(data,_id):

    try:

        collection = get_mongo_connection().Catalog.SkuDiscountInfo

        collection.update({'_id':ObjectId(_id)},{"$set":{'min_discount':data['min_discount'],'max_discount':data['max_discount'],'discountType':data['discountType'].upper().strip()}},upsert=False, multi = False)

        get_mongo_connection().Catalog.MasterData.update({'_id':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data updated successfully"}

    except:

        return {0:"Data not updated."}

def addExceptionalNlc(data):

    collection = get_mongo_connection().Catalog.ExceptionalNlc

    cursor = collection.find_one({"skuBundleId":data['skuBundleId']})

    if cursor is not None:

        return {0:"BundleId information already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def getAllExceptionlNlcItems(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.ExceptionalNlc

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def getMerchantOrdersByUser(userId, page=1, window=50, searchMap={}):

    if searchMap is None:

        searchMap = {}

    if page==None:

        page = 1

    if window==None:

        window = 50

    result = {}

    skip = (page-1)*window

    if userId is not None:

        searchMap['userId'] = userId

    collection = get_mongo_connection().Dtr.merchantOrder

    cursor = collection.find(searchMap).sort("orderId",-1)

    total_count = cursor.count()

    pages = total_count/window + (0 if total_count%window==0 else 1)  

    print "total_count", total_count

    if total_count > skip:

        cursor = cursor.skip(skip).limit(window)

        orders = []

        for order in cursor:

            del(order["_id"])

            orders.append(order)

        result['data'] = orders

        result['window'] = window

        result['totalCount'] = total_count 

        result['currCount'] = cursor.count()

        result['totalPages'] = pages

        result['currPage'] = page    

        return result

    else:

        return result

def getRefunds(userId, page=1, window=10):

    if page==None:

        page = 1

    if window==None:

        window = 10

    result = {}

    skip = (page-1)*window

    con = get_mongo_connection()

    collection = con.Dtr.refund

    user = con.Dtr.user

    credited = user.find_one({"userId":userId})

    if credited:

        credited = credited.get("credited")

    else:

        credited=0

    cursor = collection.find({"userId":userId})

    total_count = cursor.count()

    pages = total_count/window + (0 if total_count%window==0 else 1)  

    print "total_count", total_count

    if total_count > skip:

        cursor = cursor.skip(skip).limit(window).sort([('batch',-1)])

        refunds = []

        for refund in cursor:

            del(refund["_id"])

            refunds.append(refund)

        result['credited'] = credited

        result['data'] = refunds

        result['window'] = window

        result['totalCount'] = total_count 

        result['currCount'] = cursor.count()

        result['totalPages'] = pages

        result['currPage'] = page    

        return result

    else:

        return result

def getPendingRefunds(userId):

    print type(userId)

    result = get_mongo_connection().Dtr.merchantOrder\

        .aggregate([

                    {'$match':{'subOrders.cashBackStatus':CB_APPROVED, 'userId':userId}},

                    {'$unwind':"$subOrders"},

                    {'$match':{'subOrders.cashBackStatus':CB_APPROVED}},

                    { 

                     '$group':{

                               '_id':None,

                               'amount': { '$sum':'$subOrders.cashBackAmount'},

                               }

                     }

                ])['result']

    if len(result)>0:

        result = result[0]        

        result.pop("_id")

    else:

        result={}

        result['amount'] = 0.0

    result['nextCredit'] = datetime.strftime(next_weekday(datetime.now(), int(PythonPropertyReader.getConfig('CREDIT_DAY_OF_WEEK'))),"%Y-%m-%d %H:%M:%S")

    return result

def getPendingCashbacks(userId):

    result = get_mongo_connection().Dtr.merchantOrder\

        .aggregate([

                    {'$match':{'subOrders.cashBackStatus':CB_PENDING, 'userId':userId}},

                    {'$unwind':"$subOrders"},

                    {'$match':{'subOrders.cashBackStatus':CB_PENDING}},

                    { 

                     '$group':{

                               '_id':None,

                               'amount': { '$sum':'$subOrders.cashBackAmount'},

                               }

                     }

                ])['result']

    if len(result)>0:

        result = result[0]        

        result.pop("_id")

    else:

        result={}

        result['amount'] = 0.0

    return result

def __constructDummyDealObject(skuBundleId):

    temp =[]

    deals = get_mongo_connection().Catalog.Deals.find({"skuBundleId":skuBundleId,"$or":[{"showDeal":1}, {"prepaidDeal":1}]},{'_id':1,'category_id':1,'brand':1,'totalPoints':1,'bestSellerPoints':1,'nlcPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1,'subCategoryId':1})

    for d in deals:

        d['persPoints'] = 0

        temp.append(d)

    return temp

def __populateCache(userId):

    try:

        if mc.get("featured_deals") is None:

            __populateFeaturedDeals()

        featuredDeals = mc.get("featured_deals")

    except Exception as e:

        print traceback.print_exc()

        featuredDeals = {3:{},5:{},6:{}}

    print featuredDeals

    fd_bundles = []

    for catMap in featuredDeals.itervalues():

        for fd_map in catMap.itervalues():

            if not fd_map.get('skuBundleId') in fd_bundles:

                fd_bundles.append(fd_map.get('skuBundleId'))

    print "Populating memcache for userId",userId

    outer_query = []

    outer_query.append({ "$or": [ { "showDeal": 1} , { "prepaidDeal": 1 } ] })

    query = {}

    query['$gt'] = -100

    outer_query.append({'totalPoints':query})

    brandPrefMap = {}

    pricePrefMap = {}

    actionsMap = {}

    brand_p = session.query(price_preferences).filter_by(user_id=userId).all()

    for x in brand_p:

        pricePrefMap[x.category_id] = [x.min_price,x.max_price]

    for x in session.query(brand_preferences).filter_by(user_id=userId).all():

        temp_map = {}

        if brandPrefMap.has_key((x.brand).strip().upper()):

            val = brandPrefMap.get((x.brand).strip().upper())

            temp_map[x.category_id] = 1 if x.status == 'show' else 0

            val.append(temp_map)

        else:

            temp = []

            temp_map[x.category_id] = 1 if x.status == 'show' else 0

            temp.append(temp_map)

            brandPrefMap[(x.brand).strip().upper()] = temp

    for x in session.query(user_actions).filter_by(user_id=userId).all():

        actionsMap[x.store_product_id] = 1 if x.action == 'like' else 0

    all_deals = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query},{'_id':1,'category_id':1,'brand':1,'totalPoints':1,'bestSellerPoints':1,'nlcPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1, 'subCategoryId':1}).sort([('totalPoints',pymongo.DESCENDING),('bestSellerPoints',pymongo.DESCENDING),('nlcPoints',pymongo.DESCENDING),('rank',pymongo.DESCENDING)]))

    mobile_deals = []

    tablet_deals = []

    accessories_deals = []

    for deal in all_deals:

        try:

            fd_bundles.remove(deal.get('skuBundleId'))

        except:

            pass

        if actionsMap.get(deal['_id']) == 0:

            fav_weight =.25

        elif actionsMap.get(deal['_id']) == 1:

            fav_weight = 1.5

        else:

            fav_weight = 1

        if brandPrefMap.get(deal['brand'].strip().upper()) is not None:

            brand_weight = 1

            for brandInfo in brandPrefMap.get(deal['brand'].strip().upper()):

                if brandInfo.get(deal['category_id']) is not None:

                    if brandInfo.get(deal['category_id']) == 1:

                        brand_weight = 2.0

        else:

            brand_weight = 1

        if pricePrefMap.get(deal['category_id']) is not None:

            if deal['available_price'] >= pricePrefMap.get(deal['category_id'])[0] and deal['available_price'] <= pricePrefMap.get(deal['category_id'])[1]:

                asp_weight = 1.5

            elif  deal['available_price'] >= pricePrefMap.get(deal['category_id'])[0] - 0.5 * pricePrefMap.get(deal['category_id'])[0] and deal['available_price'] <= pricePrefMap.get(deal['category_id'])[1] + 0.5 * pricePrefMap.get(deal['category_id'])[1]:

                asp_weight = 1.2

            else:

                asp_weight = 1

        else:

            asp_weight = 1

        persPoints = deal['totalPoints'] * fav_weight * brand_weight * asp_weight

        deal['persPoints'] = persPoints

        if deal['category_id'] ==3:

            mobile_deals.append(deal)

        elif deal['category_id'] ==5:

            tablet_deals.append(deal)

        elif deal['category_id'] ==6:

            accessories_deals.append(deal)

        else:

            continue

    session.close()

    mobile_deals = sorted(mobile_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    tablet_deals = sorted(tablet_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    accessories_deals = sorted(accessories_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    if len(fd_bundles) > 0:

        print fd_bundles

        print "There is still bundle in feature deals with points less than -100.Lets add info"

        for skuBundleId in fd_bundles:

            dummyDealObjList = __constructDummyDealObject(skuBundleId)

            for dummyDealObj in dummyDealObjList:

                if dummyDealObj['category_id'] ==3:

                    mobile_deals.append(dummyDealObj)

                elif dummyDealObj['category_id'] ==5:

                    tablet_deals.append(dummyDealObj)

                elif dummyDealObj['category_id'] ==6:

                    accessories_deals.append(dummyDealObj)

                else:

                    pass

    for super_category, categoryFeaturedDeals in featuredDeals.iteritems():

        for val in categoryFeaturedDeals.itervalues():

            dummyDealObjList = __constructDummyDealObject(val.get('skuBundleId'))

            for dummyDealObj in dummyDealObjList:

                if super_category == dummyDealObj['category_id']:

                    continue  

                if super_category ==3:

                    mobile_deals.append(dummyDealObj)

                elif super_category ==5:

                    tablet_deals.append(dummyDealObj)

                elif super_category ==6:

                    accessories_deals.append(dummyDealObj)

                else:

                    pass

    sortedMapMobiles = {}

    rankMapMobiles = {}

    rankMobiles = 0

    blockedRanksMobiles = []

    blockedSkuBundlesMobiles = []

    blockedInfoMobiles = {}

    featuredDealsMobiles = featuredDeals.get(3)

    for k, v in featuredDealsMobiles.iteritems():

        blockedRanksMobiles.append(k-1)

        blockedSkuBundlesMobiles.append(v.get('skuBundleId'))

    for sorted_deal in mobile_deals:

        while(True):

            if rankMobiles in blockedRanksMobiles:

                print "skipping rank ",rankMobiles

                rankMobiles = rankMobiles +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesMobiles:

            print "Yes sorted deal in blocked ",sorted_deal['skuBundleId']

            if blockedInfoMobiles.has_key(sorted_deal['skuBundleId']):

                blockedInfoMobiles.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoMobiles[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapMobiles.get(sorted_deal['skuBundleId']) is None:

            sortedMapMobiles[sorted_deal['skuBundleId']] = {rankMobiles:[sorted_deal]}

            rankMapMobiles[rankMobiles] = (sortedMapMobiles[sorted_deal['skuBundleId']].values())[0]

            rankMobiles = rankMobiles +1

        else:

            for temp_list in sortedMapMobiles.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapMobiles[(sortedMapMobiles.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksMobiles:

        if blockedInfoMobiles.get((featuredDealsMobiles.get(rank+1)).get('skuBundleId')) is not None:

            rankMapMobiles[rank] = blockedInfoMobiles.get((featuredDealsMobiles.get(rank+1)).get('skuBundleId'))

            print rankMapMobiles[rank]

    sortedMapTablets = {}

    rankMapTablets = {}

    rankTablets = 0

    blockedRanksTablets = []

    blockedSkuBundlesTablets = []

    blockedInfoTablets = {}

    featuredDealsTablets = featuredDeals.get(5)

    for k, v in featuredDealsTablets.iteritems():

        blockedRanksTablets.append(k-1)

        blockedSkuBundlesTablets.append(v.get('skuBundleId'))

    for sorted_deal in tablet_deals:

        while(True):

            if rankTablets in blockedRanksTablets:

                print "skipping rank ",rankTablets

                rankTablets = rankTablets +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesTablets:

            if blockedInfoTablets.has_key(sorted_deal['skuBundleId']):

                blockedInfoTablets.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoTablets[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapTablets.get(sorted_deal['skuBundleId']) is None:

            sortedMapTablets[sorted_deal['skuBundleId']] = {rankTablets:[sorted_deal]}

            rankMapTablets[rankTablets] = (sortedMapTablets[sorted_deal['skuBundleId']].values())[0]

            rankTablets = rankTablets +1

        else:

            for temp_list in sortedMapTablets.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapTablets[(sortedMapTablets.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksTablets:

        if blockedInfoTablets.get((featuredDealsTablets.get(rank+1)).get('skuBundleId')) is not None:

            rankMapTablets[rank] = blockedInfoTablets.get((featuredDealsTablets.get(rank+1)).get('skuBundleId'))

            print rankMapTablets[rank]

    sortedMapAccessories = {}

    rankMapAccessories = {}

    rankAccessories = 0

    blockedRanksAccessories = []

    blockedSkuBundlesAccessories = []

    blockedInfoAccessories = {}

    featuredDealsAccessories = featuredDeals.get(6)

    for k, v in featuredDealsAccessories.iteritems():

        blockedRanksAccessories.append(k-1)

        blockedSkuBundlesAccessories.append(v.get('skuBundleId'))

    for sorted_deal in accessories_deals:

        while(True):

            if rankAccessories in blockedRanksAccessories:

                print "skipping rank ",rankAccessories

                rankAccessories = rankAccessories +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesAccessories:

            print "Yes sorted deal in blocked ",sorted_deal['skuBundleId']

            if blockedInfoAccessories.has_key(sorted_deal['skuBundleId']):

                blockedInfoAccessories.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoAccessories[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapAccessories.get(sorted_deal['skuBundleId']) is None:

            sortedMapAccessories[sorted_deal['skuBundleId']] = {rankAccessories:[sorted_deal]}

            rankMapAccessories[rankAccessories] = (sortedMapAccessories[sorted_deal['skuBundleId']].values())[0]

            rankAccessories = rankAccessories +1

        else:

            for temp_list in sortedMapAccessories.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapAccessories[(sortedMapAccessories.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksAccessories:

        print "Adding rank acc ",rank 

        if blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId')) is not None:

            rankMapAccessories[rank] = blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId'))

            print rankMapAccessories[rank]

    print rankMapAccessories

    mem_cache_val = {3:mobile_deals, 5:tablet_deals, 6: accessories_deals,"3_rankMap": rankMapMobiles,"5_rankMap": rankMapTablets,"6_rankMap": rankMapAccessories }

    mc.set(str(userId), mem_cache_val)

def __populateFeaturedDeals():

    print "Populating featured deals....."

    featuredDealsMobiles = {}

    featuredDealsTablets = {}

    featuredDealsAccessories = {}

    activeFeaturedDeals = get_mongo_connection().Catalog.FeaturedDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})

    for activeFeaturedDeal in activeFeaturedDeals:

        deals = get_mongo_connection().Catalog.Deals.find({"skuBundleId":activeFeaturedDeal.get('skuBundleId'),"$or":[{"showDeal":1}, {"prepaidDeal":1}]})

        for deal in deals:

            if deal.get('available_price') <= activeFeaturedDeal.get('thresholdPrice'):

                for category_id, rank  in activeFeaturedDeal['otherInfo'].iteritems():

                    f_deal = {'skuBundleId':activeFeaturedDeal.get('skuBundleId'), 'thresholdPrice':activeFeaturedDeal.get('thresholdPrice'), 'rank':rank,'category_id':int(category_id),'totalPoints':activeFeaturedDeal.get('totalPoints')}

                    if f_deal['category_id'] == 3:

                        featuredDealsMobiles[rank] = f_deal

                    elif f_deal['category_id']==5:

                        featuredDealsTablets[rank] = f_deal

                    elif f_deal['category_id']==6:

                        featuredDealsAccessories[rank] = f_deal

                break

#                f_deal = {'skuBundleId':activeFeaturedDeal.get('skuBundleId'), 'thresholdPrice':activeFeaturedDeal.get('thresholdPrice'), 'rank':activeFeaturedDeal.get('rank'),'category_id':activeFeaturedDeal.get('category_id'),'totalPoints':activeFeaturedDeal.get('totalPoints')}

#                 if activeFeaturedDeal.get('category_id') == 3:

#                     featuredDealsMobiles[activeFeaturedDeal.get('rank')] = f_deal

#                 else:

#                     featuredDealsTablets[activeFeaturedDeal.get('rank')] = f_deal

#                 break

    mc.set("featured_deals", {3:featuredDealsMobiles,5:featuredDealsTablets,6:featuredDealsAccessories}, 600)

def __populateFeaturedDealsForDealObject():

    print "Caching deal objects...."

    featuredDealObjectMobiles = {}

    featuredDealObjectTablets = {}

    featuredDealObjectAccessories = {}

    dealObjects = get_mongo_connection().Catalog.FeaturedDealsDealObject.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})

    for dealObject in dealObjects:

        d_object = list(get_mongo_connection().Catalog.DealObject.find({'_id':dealObject['_id']}))

        if len(d_object) == 0:

            continue

        for category_id, rank  in dealObject['otherInfo'].iteritems():

            d_object[0]['dealObject'] = 1

            f_deal = {'rank':rank,'category_id':int(category_id),'object':d_object[0]}

            if f_deal['category_id'] == 3:

                featuredDealObjectMobiles[rank] = f_deal

            elif f_deal['category_id']==5:

                featuredDealObjectTablets[rank] = f_deal

            elif f_deal['category_id']==6:

                featuredDealObjectAccessories[rank] = f_deal

    mc.set("featured_deals_deal_object", {3:featuredDealObjectMobiles,5:featuredDealObjectTablets,6:featuredDealObjectAccessories}, 600)

    print mc.get('featured_deals_deal_object')

def getNewDeals(userId, category_id, offset, limit, sort, direction, filterData=None):

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    dealsListMap = []

    user_specific_deals = mc.get(str(userId))

    if user_specific_deals is None:

        __populateCache(userId)

        user_specific_deals = mc.get(str(userId))

    else:

        print "Getting user deals from cache"

    category_specific_deals = user_specific_deals.get(category_id)

    insert_featured_deals = False

    if sort is None or direction is None:

        insert_featured_deals = True

        rankMap = user_specific_deals.get(str(category_id)+"_rankMap")

    else:

        if sort == "bestSellerPoints":

            sorted_deals = sorted(category_specific_deals, key = lambda x: (x['bestSellerPoints'], x['rank'], x['nlcPoints']),reverse=True)

        else:

            if direction == -1:

                rev = True

            else:

                rev = False

            sorted_deals = sorted(category_specific_deals, key = lambda x: (x['available_price']),reverse=rev)

    filtered_deals = []

    dataExist = True

    if filterData is not None:

        try:

            filtered_deals = filterDeals(category_specific_deals, filterData)

            if len(filtered_deals)==0:

                dataExist = False

        except:

            traceback.print_exc()

    if dataExist:

        if not insert_featured_deals:

            sortedMap = {}

            rankMap = {}

            rank = 0

            for sorted_deal in sorted_deals:

                if len(filtered_deals) > 0 and  sorted_deal['skuBundleId'] not in filtered_deals:

                    continue

                if sortedMap.get(sorted_deal['skuBundleId']) is None:

                    sortedMap[sorted_deal['skuBundleId']] = {rank:[sorted_deal]}

                    rankMap[rank] = (sortedMap[sorted_deal['skuBundleId']].values())[0]

                    rank = rank +1

                else:

                    for temp_list in sortedMap.get(sorted_deal['skuBundleId']).itervalues():

                        temp_list.append(sorted_deal)

                    rankMap[(sortedMap.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

        else:

            filterMap = {}

            rank = 0

            if len(filtered_deals) > 0:

                try:

                    od = collections.OrderedDict(sorted(rankMap.items()))

                except:

                    od = ordereddict.OrderedDict(sorted(rankMap.items()))

                for k, v in od.iteritems():

                    if v[0].get('skuBundleId') in filtered_deals:

                        filterMap[rank] = v

                        rank =rank+1

                rankMap = filterMap

        rankCounter = offset

        if filterData is None:

            if mc.get("featured_deals_deal_object") is None or not bool(mc.get("featured_deals_deal_object")):

                try:

                    __populateFeaturedDealsForDealObject()

                    fd_dealObject = mc.get("featured_deals_deal_object")

                except:

                    fd_dealObject = {3:{},5:{},6:{}}

            else:

                fd_dealObject = mc.get("featured_deals_deal_object")

    else:

        rankMap = {}

    for dealList in [rankMap.get(k, []) for k in range(offset, offset+limit)]:

        if filterData is None:

            while(True):

                rankCounter = rankCounter+1

                if (fd_dealObject.get(category_id).get(rankCounter)) is not None:

                    deal_object_temp = []

                    deal_object_temp.append(fd_dealObject.get(category_id).get(rankCounter).get('object'))

                    dealsListMap.append(deal_object_temp)

                else:

                    break

        temp = []

        if dealList is None or len(dealList)==0:

            print "Empty deal;lisr"

            continue

        for d in dealList:

            item = list(get_mongo_connection().Catalog.MasterData.find({'_id':d['_id']}))

            if len(item) ==0:

                continue

            item[0]['persPoints'] = d['persPoints']

            if d['dealType'] == 1 and d['source_id'] ==1:

                try:

                    manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'sku':d['_id'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))

                    item[0]['marketPlaceUrl'] = manualDeal[0]['dealUrl'].strip()

                except:

                    pass

            elif d['source_id'] ==3:

                item[0]['marketPlaceUrl'] = item[0]['marketPlaceUrl']+'?supc='+item[0].get('identifier')

            else:

                pass 

            try:

                cashBack = getCashBack(item[0]['_id'], item[0]['source_id'], item[0]['category_id'])

                if not cashBack or cashBack.get('cash_back_status')!=1:

                    item[0]['cash_back_type'] = 0

                    item[0]['cash_back'] = 0

                else:

                    if cashBack.get('maxCashBack') is not None:

                        if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*item[0]['available_price'])/100 > cashBack.get('maxCashBack'):

                            cashBack['cash_back_type'] = 2

                            cashBack['cash_back'] = cashBack['maxCashBack']

                        elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):

                            cashBack['cash_back'] = cashBack['maxCashBack']

                    item[0]['cash_back_type'] = int(cashBack['cash_back_type'])

                    item[0]['cash_back'] = cashBack['cash_back']

            except:

                print "Error in adding cashback to deals"

                item[0]['cash_back_type'] = 0

                item[0]['cash_back'] = 0

            try:

                item[0]['dp'] = d['dp']

                item[0]['showDp'] = d['showDp']

            except:

                item[0]['dp'] = 0.0

                item[0]['showDp'] = 0

            try:

                item[0]['thumbnail'] = item[0]['thumbnail'].strip()

            except:

                pass

            try:

                if item[0]['showVideo']==0:

                    item[0]['videoLink'] =""

            except:

                item[0]['videoLink'] =""

            try:

                if item[0]['quantity'] >1:

                    ppq = float(item[0]['available_price'])/item[0]['quantity']

                    if ppq == round(ppq):

                        item[0]['ppq'] = int(ppq)

                    else:

                        item[0]['ppq'] = round(ppq,2)

                else:

                    item[0]['ppq'] = 0

            except:

                item[0]['ppq'] = 0

            if filterData is None:

                if item[0]['category_id']!=6:

                    try:

                        item[0]['filterLink'] = '/category/'+str(int(item[0]['category_id']))+'?filter=brand&brands='+str(int(item[0]['brand_id']))

                        item[0]['filterText'] = 'More %s items'%(item[0]['brand'])

                    except:

                        item[0]['filterLink'] = ""

                        item[0]['filterText'] = ""

                else:

                    try:

                        if item[0]['subCategory']=='' or item[0]['subCategory'] is None or item[0]['subCategoryId']==0:

                            raise

                        item[0]['filterLink'] = '/category/'+str(int(item[0]['category_id']))+'?filter=subcategory&subcategories='+str(int(item[0]['subCategoryId']))

                        item[0]['filterText'] = 'More %s'%(item[0]['subCategory'])

                    except:

                        item[0]['filterLink'] = ''

                        item[0]['filterText'] = ""

            try:

                if item[0]['source_id']==4:

                    item_availability_info = mc.get("item_availability_"+str(item[0]['identifier']).strip())

                    if item_availability_info is None or len(item_availability_info)==0:

                        item[0]['availabilityInfo'] = [{}]

                    else:

                        item[0]['availabilityInfo'] = item_availability_info

            except:

                item[0]['availabilityInfo'] = [{}]

            temp.append(item[0])

        if len(temp) > 1:

            temp = sorted(temp, key = lambda x: (x['available_price']),reverse=False)

        dealsListMap.append(temp)

    return dealsListMap

def filterDeals(deals, filterData):

    print "Filter data ", filterData

    brandsFiltered = []

    subCategoryFiltered = []

    filterArray = filterData.split('|')

    for data in filterArray:

        try:

            filter, info = data.split(':')

        except Exception as ex:

            traceback.print_exc()

            continue

        if filter == 'brandFilter':

            toFilter = info.split('^')

            print "brand filter ",toFilter

            for deal in deals:

                if str(int(deal['brand_id'])) in toFilter:

                    brandsFiltered.append(deal)

        elif filter =='subCategoryFilter':

            toFilter = info.split('^')

            print "Sub Category filter ",toFilter

            for deal in deals:

                if str(int(deal['subCategoryId'])) in toFilter:

                    subCategoryFiltered.append(deal)

    print "No of brand filtered ",len(brandsFiltered)

    print "No of sub-cat filtered ",len(subCategoryFiltered)

    if len(subCategoryFiltered) == 0:

        return [trend['skuBundleId'] for trend in brandsFiltered]

    elif len(brandsFiltered) == 0:

        return [trend['skuBundleId'] for trend in subCategoryFiltered]

    return [i['skuBundleId'] for i in subCategoryFiltered for j in brandsFiltered if i['_id']==j['_id']]

def getDeals(userId, category_id, offset, limit, sort, direction):

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    rank = 1

    deals = {}

    outer_query = []

    outer_query.append({"showDeal":1})

    query = {}

    query['$gt'] = 0

    outer_query.append({'totalPoints':query})

    if category_id in (3,5):

        outer_query.append({'category_id':category_id})

    if sort is None or direction is None:

        direct = -1

        print outer_query

        data = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query}).sort([('totalPoints',direct),('bestSellerPoints',direct),('nlcPoints',direct),('rank',direct)]).skip(offset).limit(limit))

    else:

        print outer_query

        direct = direction

        if sort == "bestSellerPoints":

            print "yes,sorting by bestSellerPoints"

            data = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query}).sort([('bestSellerPoints',direct),('rank',direct),('nlcPoints',direct)]).skip(offset).limit(limit))

        else:

            data = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query}).sort([(sort,direct)]).skip(offset).limit(limit))

    for d in data:

        item = list(get_mongo_connection().Catalog.MasterData.find({'_id':d['_id']}))

        if not deals.has_key(item[0]['identifier']):

            item[0]['dealRank'] = rank

            try:

                cashBack = getCashBack(item[0]['_id'], item[0]['source_id'], item[0]['category_id'])

                if not cashBack or cashBack.get('cash_back_status')!=1:

                    item[0]['cash_back_type'] = 0

                    item[0]['cash_back'] = 0

                else:

                    item[0]['cash_back_type'] = int(cashBack['cash_back_type'])

                    item[0]['cash_back'] = cashBack['cash_back']

            except:

                print "Error in adding cashback to deals"

                item[0]['cash_back_type'] = 0

                item[0]['cash_back'] = 0

            deals[item[0]['identifier']] = item[0]

            rank +=1

    return sorted(deals.values(), key=itemgetter('dealRank'))

def getItem(skuId,showDp=None):

    temp = []

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    try: