Subversion Repositories SmartDukaan

from bson.objectid import ObjectId

from datetime import datetime, timedelta

from dtr.config import PythonPropertyReader

from dtr.dao import FeaturedDeals, AppTransactions, AppOfferObj

from dtr.storage import DataService

from dtr.storage.DataService import price_preferences, brand_preferences, \

    user_actions, Brands, app_offers, approved_app_transactions, user_app_cashbacks, \

    user_app_installs, appmasters, notification_campaigns

from dtr.storage.MemCache import MemCache

from dtr.utils.MailSender import Email

from dtr.utils.utils import to_java_date, CB_PENDING, CB_APPROVED, CB_INIT, \

    to_py_date, CB_REJECTED, SUB_CATEGORY_MAP

from sqlalchemy.sql.expression import func, func, or_, desc, asc, case

from elixir import *

from operator import itemgetter

from pymongo.command_cursor import CommandCursor

import pymongo

import random

import re

import time

import traceback

from itertools import groupby

import urllib

import urllib2

import json

import collections

try:

    import ordereddict

except:

    pass

con = None

DataService.initialize(db_hostname="localhost")

mc = MemCache("127.0.0.1")

SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}

COLLECTION_MAP = {

                  'ExceptionalNlc':'skuBundleId',

                  'SkuDealerPrices':'skuBundleId',

                  'SkuDiscountInfo':'skuBundleId',

                  'SkuSchemeDetails':'skuBundleId',

                  'DealPoints':'skuBundleId',

                  'FeaturedDeals': 'skuBundleId'

                  }

def get_mongo_connection(host='localhost', port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def populateCashBack():

    print "Populating cashback"

    cashBackMap = {}

    itemCashBackMap = {}

    cashBack = list(get_mongo_connection().Catalog.CategoryCashBack.find())

    for row in cashBack:

        temp_map = {}

        temp_list = []

        if cashBackMap.has_key(row['source_id']):

            arr = cashBackMap.get(row['source_id'])

            for val in arr:

                temp_list.append(val)

            temp_map[row['category_id']] = row

            temp_list.append(temp_map)

            cashBackMap[row['source_id']] = temp_list 

        else:

            temp_map[row['category_id']] = row

            temp_list.append(temp_map)

            cashBackMap[row['source_id']] = temp_list

    itemCashBack = list(get_mongo_connection().Catalog.ItemCashBack.find())

    for row in itemCashBack:

        if not itemCashBackMap.has_key(row['skuId']):

            itemCashBackMap[row['skuId']] = row

    mc.set("item_cash_back", itemCashBackMap, 24 * 60 * 60)

    mc.set("category_cash_back", cashBackMap, 24 * 60 * 60)

def addCategoryDiscount(data):

    collection = get_mongo_connection().Catalog.CategoryDiscount

    query = []

    data['brand'] = data['brand'].strip().upper()

    data['discountType'] = data['discountType'].upper().strip()

    query.append({"brand":data['brand']})

    query.append({"category_id":data['category_id']})

    r = collection.find({"$and":query})

    if r.count() > 0:

        return {0:"Brand & Category info already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'brand':data['brand'],'category_id':data['category_id']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def updateCategoryDiscount(data,_id):

    try:

        collection = get_mongo_connection().Catalog.CategoryDiscount

        collection.update({'_id':ObjectId(_id)},{"$set":{'min_discount':data['min_discount'],'max_discount':data['max_discount'],'discountType':data['discountType'].upper().strip()}},upsert=False, multi = False)

        get_mongo_connection().Catalog.MasterData.update({'brand':data['brand'],'category_id':data['category_id']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data updated successfully"}

    except:

        return {0:"Data not updated."}

def getAllCategoryDiscount():

    data = []

    collection = get_mongo_connection().Catalog.CategoryDiscount

    cursor = collection.find()

    for val in cursor:

        data.append(val)

    return data

def __getBundledSkusfromSku(sku):

    masterData =  get_mongo_connection().Catalog.MasterData.find_one({"_id":sku},{"skuBundleId":1})

    if masterData is not None:

        return list(get_mongo_connection().Catalog.MasterData.find({"skuBundleId":masterData.get('skuBundleId')},{'_id':1,'skuBundleId':1}))

    else:

        return []

def addSchemeDetailsForSku(data):

    collection = get_mongo_connection().Catalog.SkuSchemeDetails

    result = collection.find_one({'skuBundleId':data['skuBundleId']})

    if result is None:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

    return {1:"BundleId info already present"}

def getAllSkuWiseSchemeDetails(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.SkuSchemeDetails

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def addSkuDiscountInfo(data):

    collection = get_mongo_connection().Catalog.SkuDiscountInfo

    cursor = collection.find_one({"skuBundleId":data['skuBundleId']})

    if cursor is not None:

        return {0:"BundleId information already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def getallSkuDiscountInfo(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.SkuDiscountInfo

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def updateSkuDiscount(data,_id):

    try:

        collection = get_mongo_connection().Catalog.SkuDiscountInfo

        collection.update({'_id':ObjectId(_id)},{"$set":{'min_discount':data['min_discount'],'max_discount':data['max_discount'],'discountType':data['discountType'].upper().strip()}},upsert=False, multi = False)

        get_mongo_connection().Catalog.MasterData.update({'_id':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data updated successfully"}

    except:

        return {0:"Data not updated."}

def addExceptionalNlc(data):

    collection = get_mongo_connection().Catalog.ExceptionalNlc

    cursor = collection.find_one({"skuBundleId":data['skuBundleId']})

    if cursor is not None:

        return {0:"BundleId information already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def getAllExceptionlNlcItems(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.ExceptionalNlc

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def getMerchantOrdersByUser(userId, page=1, window=50, searchMap={}):

    if searchMap is None:

        searchMap = {}

    if page==None:

        page = 1

    if window==None:

        window = 50

    result = {}

    skip = (page-1)*window

    if userId is not None:

        searchMap['userId'] = userId

    collection = get_mongo_connection().Dtr.merchantOrder

    cursor = collection.find(searchMap).sort("orderId",-1)

    total_count = cursor.count()

    pages = total_count/window + (0 if total_count%window==0 else 1)  

    print "total_count", total_count

    if total_count > skip:

        cursor = cursor.skip(skip).limit(window)

        orders = []

        for order in cursor:

            del(order["_id"])

            orders.append(order)

        result['data'] = orders

        result['window'] = window

        result['totalCount'] = total_count 

        result['currCount'] = cursor.count()

        result['totalPages'] = pages

        result['currPage'] = page    

        return result

    else:

        return result

def getRefunds(userId, page=1, window=10):

    if page==None:

        page = 1

    if window==None:

        window = 10

    result = {}

    skip = (page-1)*window

    con = get_mongo_connection()

    collection = con.Dtr.refund

    user = con.Dtr.user

    credited = user.find_one({"userId":userId})

    if credited:

        credited = credited.get("credited")

    else:

        credited=0

    cursor = collection.find({"userId":userId})

    total_count = cursor.count()

    pages = total_count/window + (0 if total_count%window==0 else 1)  

    print "total_count", total_count

    if total_count > skip:

        cursor = cursor.skip(skip).limit(window).sort([('batch',-1)])

        refunds = []

        for refund in cursor:

            del(refund["_id"])

            refunds.append(refund)

        result['credited'] = credited

        result['data'] = refunds

        result['window'] = window

        result['totalCount'] = total_count 

        result['currCount'] = cursor.count()

        result['totalPages'] = pages

        result['currPage'] = page    

        return result

    else:

        return result

def getPendingRefunds(userId):

    print type(userId)

    result = get_mongo_connection().Dtr.merchantOrder\

        .aggregate([

                    {'$match':{'subOrders.cashBackStatus':CB_APPROVED, 'userId':userId}},

                    {'$unwind':"$subOrders"},

                    {'$match':{'subOrders.cashBackStatus':CB_APPROVED}},

                    { 

                     '$group':{

                               '_id':None,

                               'amount': { '$sum':'$subOrders.cashBackAmount'},

                               }

                     }

                ])['result']

    if len(result)>0:

        result = result[0]        

        result.pop("_id")

    else:

        result={}

        result['amount'] = 0.0

    result['nextCredit'] = datetime.strftime(next_weekday(datetime.now(), int(PythonPropertyReader.getConfig('CREDIT_DAY_OF_WEEK'))),"%Y-%m-%d %H:%M:%S")

    return result

def getPendingCashbacks(userId):

    result = get_mongo_connection().Dtr.merchantOrder\

        .aggregate([

                    {'$match':{'subOrders.cashBackStatus':CB_PENDING, 'userId':userId}},

                    {'$unwind':"$subOrders"},

                    {'$match':{'subOrders.cashBackStatus':CB_PENDING}},

                    { 

                     '$group':{

                               '_id':None,

                               'amount': { '$sum':'$subOrders.cashBackAmount'},

                               }

                     }

                ])['result']

    if len(result)>0:

        result = result[0]        

        result.pop("_id")

    else:

        result={}

        result['amount'] = 0.0

    return result

def __constructDummyDealObject(skuBundleId):

    temp =[]

    deals = get_mongo_connection().Catalog.Deals.find({"skuBundleId":skuBundleId,"$or":[{"showDeal":1}, {"prepaidDeal":1}]},{'_id':1,'category_id':1,'brand':1,'totalPoints':1,'bestSellerPoints':1,'nlcPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1})

    for d in deals:

        d['persPoints'] = 0

        temp.append(d)

    return temp

def __populateCache(userId):

    try:

        if mc.get("featured_deals") is None:

            __populateFeaturedDeals()

        featuredDeals = mc.get("featured_deals")

    except Exception as e:

        print traceback.print_exc()

        featuredDeals = {3:{},5:{},6:{}}

    print featuredDeals

    fd_bundles = []

    for catMap in featuredDeals.itervalues():

        for fd_map in catMap.itervalues():

            if not fd_map.get('skuBundleId') in fd_bundles:

                fd_bundles.append(fd_map.get('skuBundleId'))

    print "Populating memcache for userId",userId

    outer_query = []

    outer_query.append({ "$or": [ { "showDeal": 1} , { "prepaidDeal": 1 } ] })

    query = {}

    query['$gt'] = -100

    outer_query.append({'totalPoints':query})

    brandPrefMap = {}

    pricePrefMap = {}

    actionsMap = {}

    brand_p = session.query(price_preferences).filter_by(user_id=userId).all()

    for x in brand_p:

        pricePrefMap[x.category_id] = [x.min_price,x.max_price]

    for x in session.query(brand_preferences).filter_by(user_id=userId).all():

        temp_map = {}

        if brandPrefMap.has_key((x.brand).strip().upper()):

            val = brandPrefMap.get((x.brand).strip().upper())

            temp_map[x.category_id] = 1 if x.status == 'show' else 0

            val.append(temp_map)

        else:

            temp = []

            temp_map[x.category_id] = 1 if x.status == 'show' else 0

            temp.append(temp_map)

            brandPrefMap[(x.brand).strip().upper()] = temp

    for x in session.query(user_actions).filter_by(user_id=userId).all():

        actionsMap[x.store_product_id] = 1 if x.action == 'like' else 0

    all_deals = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query},{'_id':1,'category_id':1,'brand':1,'totalPoints':1,'bestSellerPoints':1,'nlcPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1}).sort([('totalPoints',pymongo.DESCENDING),('bestSellerPoints',pymongo.DESCENDING),('nlcPoints',pymongo.DESCENDING),('rank',pymongo.DESCENDING)]))

    mobile_deals = []

    tablet_deals = []

    accessories_deals = []

    for deal in all_deals:

        try:

            fd_bundles.remove(deal.get('skuBundleId'))

        except:

            pass

        if actionsMap.get(deal['_id']) == 0:

            fav_weight =.25

        elif actionsMap.get(deal['_id']) == 1:

            fav_weight = 1.5

        else:

            fav_weight = 1

        if brandPrefMap.get(deal['brand'].strip().upper()) is not None:

            brand_weight = 1

            for brandInfo in brandPrefMap.get(deal['brand'].strip().upper()):

                if brandInfo.get(deal['category_id']) is not None:

                    if brandInfo.get(deal['category_id']) == 1:

                        brand_weight = 2.0

        else:

            brand_weight = 1

        if pricePrefMap.get(deal['category_id']) is not None:

            if deal['available_price'] >= pricePrefMap.get(deal['category_id'])[0] and deal['available_price'] <= pricePrefMap.get(deal['category_id'])[1]:

                asp_weight = 1.5

            elif  deal['available_price'] >= pricePrefMap.get(deal['category_id'])[0] - 0.5 * pricePrefMap.get(deal['category_id'])[0] and deal['available_price'] <= pricePrefMap.get(deal['category_id'])[1] + 0.5 * pricePrefMap.get(deal['category_id'])[1]:

                asp_weight = 1.2

            else:

                asp_weight = 1

        else:

            asp_weight = 1

        persPoints = deal['totalPoints'] * fav_weight * brand_weight * asp_weight

        deal['persPoints'] = persPoints

        if deal['category_id'] ==3:

            mobile_deals.append(deal)

        elif deal['category_id'] ==5:

            tablet_deals.append(deal)

        elif deal['category_id'] ==6:

            accessories_deals.append(deal)

        else:

            continue

    session.close()

    mobile_deals = sorted(mobile_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    tablet_deals = sorted(tablet_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    accessories_deals = sorted(accessories_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    if len(fd_bundles) > 0:

        print fd_bundles

        print "There is still bundle in feature deals with points less than -100.Lets add info"

        for skuBundleId in fd_bundles:

            dummyDealObjList = __constructDummyDealObject(skuBundleId)

            for dummyDealObj in dummyDealObjList:

                if dummyDealObj['category_id'] ==3:

                    mobile_deals.append(dummyDealObj)

                elif dummyDealObj['category_id'] ==5:

                    tablet_deals.append(dummyDealObj)

                elif dummyDealObj['category_id'] ==6:

                    accessories_deals.append(dummyDealObj)

                else:

                    pass

    for super_category, categoryFeaturedDeals in featuredDeals.iteritems():

        for val in categoryFeaturedDeals.itervalues():

            dummyDealObjList = __constructDummyDealObject(val.get('skuBundleId'))

            for dummyDealObj in dummyDealObjList:

                if super_category == dummyDealObj['category_id']:

                    continue  

                if super_category ==3:

                    mobile_deals.append(dummyDealObj)

                elif super_category ==5:

                    tablet_deals.append(dummyDealObj)

                elif super_category ==6:

                    accessories_deals.append(dummyDealObj)

                else:

                    pass

    sortedMapMobiles = {}

    rankMapMobiles = {}

    rankMobiles = 0

    blockedRanksMobiles = []

    blockedSkuBundlesMobiles = []

    blockedInfoMobiles = {}

    featuredDealsMobiles = featuredDeals.get(3)

    for k, v in featuredDealsMobiles.iteritems():

        blockedRanksMobiles.append(k-1)

        blockedSkuBundlesMobiles.append(v.get('skuBundleId'))

    print "Blocked ranks ", blockedRanksMobiles

    print "Blocked bundles ", blockedSkuBundlesMobiles 

    for sorted_deal in mobile_deals:

        while(True):

            if rankMobiles in blockedRanksMobiles:

                print "skipping rank ",rankMobiles

                rankMobiles = rankMobiles +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesMobiles:

            print "Yes sorted deal in blocked ",sorted_deal['skuBundleId']

            if blockedInfoMobiles.has_key(sorted_deal['skuBundleId']):

                blockedInfoMobiles.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoMobiles[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapMobiles.get(sorted_deal['skuBundleId']) is None:

            sortedMapMobiles[sorted_deal['skuBundleId']] = {rankMobiles:[sorted_deal]}

            rankMapMobiles[rankMobiles] = (sortedMapMobiles[sorted_deal['skuBundleId']].values())[0]

            rankMobiles = rankMobiles +1

        else:

            for temp_list in sortedMapMobiles.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapMobiles[(sortedMapMobiles.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    print "Blocked info ", blockedInfoMobiles

    for rank in blockedRanksMobiles:

        print "Adding rank ",rank 

        if blockedInfoMobiles.get((featuredDealsMobiles.get(rank+1)).get('skuBundleId')) is not None:

            rankMapMobiles[rank] = blockedInfoMobiles.get((featuredDealsMobiles.get(rank+1)).get('skuBundleId'))

            print rankMapMobiles[rank]

    sortedMapTablets = {}

    rankMapTablets = {}

    rankTablets = 0

    blockedRanksTablets = []

    blockedSkuBundlesTablets = []

    blockedInfoTablets = {}

    featuredDealsTablets = featuredDeals.get(5)

    for k, v in featuredDealsTablets.iteritems():

        blockedRanksTablets.append(k-1)

        blockedSkuBundlesTablets.append(v.get('skuBundleId'))

    print "Blocked ranks ", blockedRanksTablets

    print "Blocked bundles ", blockedSkuBundlesTablets 

    for sorted_deal in tablet_deals:

        while(True):

            if rankTablets in blockedRanksTablets:

                print "skipping rank ",rankTablets

                rankTablets = rankTablets +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesTablets:

            if blockedInfoTablets.has_key(sorted_deal['skuBundleId']):

                blockedInfoTablets.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoTablets[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapTablets.get(sorted_deal['skuBundleId']) is None:

            sortedMapTablets[sorted_deal['skuBundleId']] = {rankTablets:[sorted_deal]}

            rankMapTablets[rankTablets] = (sortedMapTablets[sorted_deal['skuBundleId']].values())[0]

            rankTablets = rankTablets +1

        else:

            for temp_list in sortedMapTablets.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapTablets[(sortedMapTablets.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksTablets:

        print "Adding rank ",rank 

        if blockedInfoTablets.get((featuredDealsTablets.get(rank+1)).get('skuBundleId')) is not None:

            rankMapTablets[rank] = blockedInfoTablets.get((featuredDealsTablets.get(rank+1)).get('skuBundleId'))

            print rankMapTablets[rank]

    sortedMapAccessories = {}

    rankMapAccessories = {}

    rankAccessories = 0

    blockedRanksAccessories = []

    blockedSkuBundlesAccessories = []

    blockedInfoAccessories = {}

    featuredDealsAccessories = featuredDeals.get(6)

    for k, v in featuredDealsAccessories.iteritems():

        blockedRanksAccessories.append(k-1)

        blockedSkuBundlesAccessories.append(v.get('skuBundleId'))

    print "Blocked ranks ", blockedRanksAccessories

    print "Blocked bundles ", blockedSkuBundlesAccessories 

    for sorted_deal in accessories_deals:

        while(True):

            if rankAccessories in blockedRanksAccessories:

                print "skipping rank ",rankAccessories

                rankAccessories = rankAccessories +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesAccessories:

            print "Yes sorted deal in blocked ",sorted_deal['skuBundleId']

            if blockedInfoAccessories.has_key(sorted_deal['skuBundleId']):

                blockedInfoAccessories.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoAccessories[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapAccessories.get(sorted_deal['skuBundleId']) is None:

            sortedMapAccessories[sorted_deal['skuBundleId']] = {rankAccessories:[sorted_deal]}

            rankMapAccessories[rankAccessories] = (sortedMapAccessories[sorted_deal['skuBundleId']].values())[0]

            rankAccessories = rankAccessories +1

        else:

            for temp_list in sortedMapAccessories.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapAccessories[(sortedMapAccessories.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    print "Blocked info ", blockedInfoAccessories

    for rank in blockedRanksAccessories:

        print "Adding rank acc ",rank 

        if blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId')) is not None:

            rankMapAccessories[rank] = blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId'))

            print rankMapAccessories[rank]

    print rankMapAccessories

    mem_cache_val = {3:mobile_deals, 5:tablet_deals, 6: accessories_deals,"3_rankMap": rankMapMobiles,"5_rankMap": rankMapTablets,"6_rankMap": rankMapAccessories }

    mc.set(str(userId), mem_cache_val)

def __populateFeaturedDeals():

    print "Populating featured deals....."

    featuredDealsMobiles = {}

    featuredDealsTablets = {}

    featuredDealsAccessories = {}

    activeFeaturedDeals = get_mongo_connection().Catalog.FeaturedDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}).sort([('rank',pymongo.DESCENDING)])

    for activeFeaturedDeal in activeFeaturedDeals:

        deals = get_mongo_connection().Catalog.Deals.find({"skuBundleId":activeFeaturedDeal.get('skuBundleId'),"$or":[{"showDeal":1}, {"prepaidDeal":1}]})

        for deal in deals:

            if deal.get('available_price') <= activeFeaturedDeal.get('thresholdPrice'):

                for category_id, rank  in activeFeaturedDeal['otherInfo'].iteritems():

                    f_deal = {'skuBundleId':activeFeaturedDeal.get('skuBundleId'), 'thresholdPrice':activeFeaturedDeal.get('thresholdPrice'), 'rank':rank,'category_id':int(category_id),'totalPoints':activeFeaturedDeal.get('totalPoints')}

                    if f_deal['category_id'] == 3:

                        featuredDealsMobiles[rank] = f_deal

                    elif f_deal['category_id']==5:

                        featuredDealsTablets[rank] = f_deal

                    elif f_deal['category_id']==6:

                        featuredDealsAccessories[rank] = f_deal

                break

#                f_deal = {'skuBundleId':activeFeaturedDeal.get('skuBundleId'), 'thresholdPrice':activeFeaturedDeal.get('thresholdPrice'), 'rank':activeFeaturedDeal.get('rank'),'category_id':activeFeaturedDeal.get('category_id'),'totalPoints':activeFeaturedDeal.get('totalPoints')}

#                 if activeFeaturedDeal.get('category_id') == 3:

#                     featuredDealsMobiles[activeFeaturedDeal.get('rank')] = f_deal

#                 else:

#                     featuredDealsTablets[activeFeaturedDeal.get('rank')] = f_deal

#                 break

    mc.set("featured_deals", {3:featuredDealsMobiles,5:featuredDealsTablets,6:featuredDealsAccessories}, 600)

def getNewDeals(userId, category_id, offset, limit, sort, direction, filterData=None):

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    dealsListMap = []

    user_specific_deals = mc.get(str(userId))

    if user_specific_deals is None:

        __populateCache(userId)

        user_specific_deals = mc.get(str(userId))

    else:

        print "Getting user deals from cache"

    category_specific_deals = user_specific_deals.get(category_id)

    insert_featured_deals = False

    if sort is None or direction is None:

        insert_featured_deals = True

        rankMap = user_specific_deals.get(str(category_id)+"_rankMap")

    else:

        if sort == "bestSellerPoints":

            sorted_deals = sorted(category_specific_deals, key = lambda x: (x['bestSellerPoints'], x['rank'], x['nlcPoints']),reverse=True)

        else:

            if direction == -1:

                rev = True

            else:

                rev = False

            sorted_deals = sorted(category_specific_deals, key = lambda x: (x['available_price']),reverse=rev)

    print "============================"

    filtered_deals = []

    if filterData is not None:

        try:

            filtered_deals = filterDeals(category_specific_deals, filterData)

        except:

            traceback.print_exc()

    if not insert_featured_deals:

        sortedMap = {}

        rankMap = {}

        rank = 0

        for sorted_deal in sorted_deals:

            if len(filtered_deals) > 0 and  sorted_deal['skuBundleId'] not in filtered_deals:

                continue

            if sortedMap.get(sorted_deal['skuBundleId']) is None:

                sortedMap[sorted_deal['skuBundleId']] = {rank:[sorted_deal]}

                rankMap[rank] = (sortedMap[sorted_deal['skuBundleId']].values())[0]

                rank = rank +1

            else:

                for temp_list in sortedMap.get(sorted_deal['skuBundleId']).itervalues():

                    temp_list.append(sorted_deal)

                rankMap[(sortedMap.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    else:

        filterMap = {}

        rank = 0

        if len(filtered_deals) > 0:

            try:

                od = collections.OrderedDict(sorted(rankMap.items()))

            except:

                od = ordereddict.OrderedDict(sorted(rankMap.items()))

            for k, v in od.iteritems():

                if v[0].get('skuBundleId') in filtered_deals:

                    filterMap[rank] = v

                    rank =rank+1

            rankMap = filterMap

    for dealList in [rankMap.get(k, []) for k in range(offset, offset+limit)]:

        temp = []

        if dealList is None or len(dealList)==0:

            print "Empty deal;lisr"

            continue

        for d in dealList:

            item = list(get_mongo_connection().Catalog.MasterData.find({'_id':d['_id']}))

            if len(item) ==0:

                continue

            item[0]['persPoints'] = d['persPoints']

            if d['dealType'] == 1 and d['source_id'] ==1:

                try:

                    manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'sku':d['_id'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))

                    item[0]['marketPlaceUrl'] = manualDeal[0]['dealUrl'].strip()

                except:

                    pass

            elif d['source_id'] ==3:

                item[0]['marketPlaceUrl'] = item[0]['marketPlaceUrl']+'?supc='+item[0].get('identifier')

            else:

                pass 

            try:

                cashBack = getCashBack(item[0]['_id'], item[0]['source_id'], item[0]['category_id'])

                if not cashBack or cashBack.get('cash_back_status')!=1:

                    item[0]['cash_back_type'] = 0

                    item[0]['cash_back'] = 0

                else:

                    if cashBack.get('maxCashBack') is not None:

                        if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*item[0]['available_price'])/100 > cashBack.get('maxCashBack'):

                            cashBack['cash_back_type'] = 2

                            cashBack['cash_back'] = cashBack['maxCashBack']

                        elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):

                            cashBack['cash_back'] = cashBack['maxCashBack']

                    item[0]['cash_back_type'] = int(cashBack['cash_back_type'])

                    item[0]['cash_back'] = cashBack['cash_back']

            except:

                print "Error in adding cashback to deals"

                item[0]['cash_back_type'] = 0

                item[0]['cash_back'] = 0

            try:

                item[0]['dp'] = d['dp']

                item[0]['showDp'] = d['showDp']

            except:

                item[0]['dp'] = 0.0

                item[0]['showDp'] = 0

            try:

                item[0]['thumbnail'] = item[0]['thumbnail'].strip()

            except:

                pass

            try:

                if item[0]['showVideo']==0:

                    item[0]['videoLink'] =""

            except:

                item[0]['videoLink'] =""

            try:

                if item[0]['quantity'] >1:

                    ppq = float(item[0]['available_price'])/item[0]['quantity']

                    if ppq == round(ppq):

                        item[0]['ppq'] = int(ppq)

                    else:

                        item[0]['ppq'] = round(ppq,2)

                    item[0]['tagline'] = ""

                else:

                    item[0]['ppq'] = 0

            except:

                item[0]['ppq'] = 0

            try:

                item[0]['filterLink'] = '/category/'+str(int(item[0]['category_id']))+'?filter=brand&brands='+str(int(item[0]['brand_id']))

            except:

                item[0]['filterLink'] = ""

            temp.append(item[0])

        if len(temp) > 1:

            temp = sorted(temp, key = lambda x: (x['available_price']),reverse=False)

        dealsListMap.append(temp)

    return dealsListMap

def filterDeals(deals, filterData):

    dealFiltered = []

    brandsFiltered = []

    filterArray = filterData.split('|')

    for data in filterArray:

        try:

            filter, info = data.split(':')

        except Exception as ex:

            traceback.print_exc()

            continue

        if filter == 'dealFilter':

            toFilter = info.split('^')

            print "deal filter ",toFilter

            if 'deals' in toFilter:

                dealFiltered = deals

                continue

            for filterVal in toFilter:

                if filterVal == 'dod':

                    for deal in deals:

                        if deal['dealType'] == 1:

                            dealFiltered.append(deal)

        elif filter == 'brandFilter':

            toFilter = info.split('^')

            print "brand filter ",toFilter

            if len(toFilter) == 0 or (len(toFilter)==1 and toFilter[0]==''):

                brandsFiltered = deals

            for deal in deals:

                if str(int(deal['brand_id'])) in toFilter:

                    brandsFiltered.append(deal)

    if len(dealFiltered) == 0:

        return [trend['skuBundleId'] for trend in brandsFiltered]

    return [i['skuBundleId'] for i in dealFiltered for j in brandsFiltered if i['_id']==j['_id']]

def getDeals(userId, category_id, offset, limit, sort, direction):

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    rank = 1

    deals = {}

    outer_query = []

    outer_query.append({"showDeal":1})

    query = {}

    query['$gt'] = 0

    outer_query.append({'totalPoints':query})

    if category_id in (3,5):

        outer_query.append({'category_id':category_id})

    if sort is None or direction is None:

        direct = -1

        print outer_query

        data = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query}).sort([('totalPoints',direct),('bestSellerPoints',direct),('nlcPoints',direct),('rank',direct)]).skip(offset).limit(limit))

    else:

        print outer_query

        direct = direction

        if sort == "bestSellerPoints":

            print "yes,sorting by bestSellerPoints"

            data = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query}).sort([('bestSellerPoints',direct),('rank',direct),('nlcPoints',direct)]).skip(offset).limit(limit))

        else:

            data = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query}).sort([(sort,direct)]).skip(offset).limit(limit))

    for d in data:

        item = list(get_mongo_connection().Catalog.MasterData.find({'_id':d['_id']}))

        if not deals.has_key(item[0]['identifier']):

            item[0]['dealRank'] = rank

            try:

                cashBack = getCashBack(item[0]['_id'], item[0]['source_id'], item[0]['category_id'])

                if not cashBack or cashBack.get('cash_back_status')!=1:

                    item[0]['cash_back_type'] = 0

                    item[0]['cash_back'] = 0

                else:

                    item[0]['cash_back_type'] = int(cashBack['cash_back_type'])

                    item[0]['cash_back'] = cashBack['cash_back']

            except:

                print "Error in adding cashback to deals"

                item[0]['cash_back_type'] = 0

                item[0]['cash_back'] = 0

            deals[item[0]['identifier']] = item[0]

            rank +=1

    return sorted(deals.values(), key=itemgetter('dealRank'))

def getItem(skuId,showDp=None):

    temp = []

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    try:

        skuData = get_mongo_connection().Catalog.MasterData.find_one({'_id':int(skuId)})

        if skuData is None:

            raise

        try:

            cashBack = getCashBack(skuData['_id'], skuData['source_id'], skuData['category_id'])

            if not cashBack or cashBack.get('cash_back_status')!=1:

                skuData['cash_back_type'] = 0

                skuData['cash_back'] = 0

            else:

                skuData['cash_back_type'] = cashBack['cash_back_type']

                skuData['cash_back'] = cashBack['cash_back']

        except:

            print "Error in adding cashback to deals"

            skuData['cash_back_type'] = 0

            skuData['cash_back'] = 0

        skuData['in_stock'] = int(skuData['in_stock'])

        skuData['is_shortage'] = int(skuData['is_shortage'])

        skuData['category_id'] = int(skuData['category_id'])

        skuData['status'] = int(skuData['status'])

        try:

            skuData['thumbnail'] = skuData['thumbnail'].strip()

        except:

            pass 

        if showDp is not None:

            dealerPrice = get_mongo_connection().Catalog.SkuDealerPrices.find_one({'skuBundleId':skuData['skuBundleId']})

            skuData['dp'] = 0.0

            skuData['showDp'] = 0

            if dealerPrice is not None:

                skuData['dp'] = dealerPrice['dp']

                skuData['showDp'] = dealerPrice['showDp']

        temp.append(skuData)

        return temp

    except:

        return []

def getCashBack(skuId, source_id, category_id):

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    itemCashBackMap = mc.get("item_cash_back")

    itemCashBack = itemCashBackMap.get(skuId)

    if itemCashBack is not None:

        return itemCashBack

    cashBackMap = mc.get("category_cash_back")

    sourceCashBack = cashBackMap.get(source_id)

    if sourceCashBack is not None and len(sourceCashBack) > 0:

        for cashBack in sourceCashBack:

            if cashBack.get(category_id) is None:

                continue

            else:

                return cashBack.get(category_id)

    else:

        return {}

def getBundleBySourceSku(source_id, identifier):

    if source_id in (1,2,4,5,6,7):

        skuData = list(get_mongo_connection().Catalog.MasterData.find({'identifier':identifier.strip(), 'source_id':source_id}))

    elif source_id == 3:

        skuData = list(get_mongo_connection().Catalog.MasterData.find({'secondaryIdentifier':identifier.strip(), 'source_id':source_id}))

    else:

        return {}

    if not skuData:

        return {}

    else:

        bundleId = skuData[0]["skuBundleId"]

        itemIds = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bundleId, 'in_stock':1, 'source_id':{"$in":[1,2,3,5,6,7]}}, 

                       {"source_id":1, "available_price":1, "marketPlaceUrl":1, "gross_price":1, "codAvailable":1, "source_product_name":1, "offer":1, "coupon":1}))

        return {'products':itemIds}

def getDealRank(identifier, source_id, userId):

    if source_id in (1,2,4,5,6,7):

        skuData = list(get_mongo_connection().Catalog.MasterData.find({'identifier':identifier.strip(), 'source_id':source_id}))

    elif source_id == 3:

        skuData = list(get_mongo_connection().Catalog.MasterData.find({'secondaryIdentifier':identifier.strip(), 'source_id':source_id}))

    else:

        return {'rank':0, 'description':'Source not valid','maxNlc':None, 'minNlc':None,'status':None,'dp':None}