Subversion Repositories SmartDukaan

from bson.objectid import ObjectId

from datetime import datetime, timedelta

from dtr.config import PythonPropertyReader

from dtr.dao import FeaturedDeals, AppTransactions, AppOfferObj, Promotion, NotificationCampaign,CrmRefundWallet

from dtr.storage import DataService

from dtr.storage.DataService import price_preferences, brand_preferences, \

    user_actions, Brands, app_offers, approved_app_transactions, user_app_cashbacks, \

    user_app_installs, appmasters, notification_campaigns, user_accounts

from dtr.storage.MemCache import MemCache

from dtr.utils.MailSender import Email

from dtr.utils.utils import to_java_date, CB_PENDING, CB_APPROVED, CB_INIT, \

    to_py_date, CB_REJECTED, SUB_CATEGORY_MAP,CREDIT_TYPE_REFUND,CREDIT_TYPE_OFFER,CREDIT_TYPE_ADJUSTMENT,REFUND_ADJUSTMENT_MAP, todict,REVERSE_SOURCE_MAP,\

    get_mongo_connection_dtr_data, SUB_CATEGORY_HEADER_RANKING

from sqlalchemy.sql.expression import func, func, or_, desc, asc, case

from elixir import *

from operator import itemgetter

from pymongo.command_cursor import CommandCursor

import pymongo

import random

import re

import time

import traceback

from itertools import groupby

import urllib

import urllib2

import json

import collections

from string import lower

try:

    import ordereddict

except:

    pass

import bson.son as son

import math

con = None

es = None

try:

    from elasticsearch import Elasticsearch

except:

    pass

from shop2020.config.client.ConfigClient import ConfigClient

DataService.initialize()

mc = MemCache("127.0.0.1")

config_client = ConfigClient()

elastic_search_host = config_client.get_property('elastic_search_host')

elastic_search_port = config_client.get_property('elastic_search_port')

SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}

ONLINE_DEAL_SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}

COLLECTION_MAP = {

                  'ExceptionalNlc':'skuBundleId',

                  'SkuDealerPrices':'skuBundleId',

                  'SkuDiscountInfo':'skuBundleId',

                  'SkuSchemeDetails':'skuBundleId',

                  'DealPoints':'skuBundleId',

                  'FeaturedDeals': 'skuBundleId'

                  }

def get_elastic_search_connection():

    global es

    if es is None:

        print "Establishing connection with elastic search %s host and port %s"%(elastic_search_host,elastic_search_port)

        es = Elasticsearch([{'host': elastic_search_host, 'port': elastic_search_port}])

        return es

    return es

def get_mongo_connection(host='localhost', port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def populateCashBack():

    print "Populating cashback"

    cashBackMap = {}

    itemCashBackMap = {}

    cashBack = list(get_mongo_connection().Catalog.CategoryCashBack.find())

    for row in cashBack:

        temp_map = {}

        temp_list = []

        if cashBackMap.has_key(row['source_id']):

            arr = cashBackMap.get(row['source_id'])

            for val in arr:

                temp_list.append(val)

            temp_map[row['category_id']] = row

            temp_list.append(temp_map)

            cashBackMap[row['source_id']] = temp_list 

        else:

            temp_map[row['category_id']] = row

            temp_list.append(temp_map)

            cashBackMap[row['source_id']] = temp_list

    itemCashBack = list(get_mongo_connection().Catalog.ItemCashBack.find())

    for row in itemCashBack:

        if not itemCashBackMap.has_key(row['sku']):

            itemCashBackMap[row['sku']] = row

    mc.set("item_cash_back", itemCashBackMap, 24 * 60 * 60)

    mc.set("category_cash_back", cashBackMap, 24 * 60 * 60)

def addCategoryDiscount(data):

    collection = get_mongo_connection().Catalog.CategoryDiscount

    query = []

    data['brand'] = data['brand'].strip().upper()

    data['discountType'] = data['discountType'].upper().strip()

    query.append({"brand":data['brand']})

    query.append({"category_id":data['category_id']})

    r = collection.find({"$and":query})

    if r.count() > 0:

        return {0:"Brand & Category info already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'brand':data['brand'],'category_id':data['category_id']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def updateCategoryDiscount(data,_id):

    try:

        collection = get_mongo_connection().Catalog.CategoryDiscount

        collection.update({'_id':ObjectId(_id)},{"$set":{'min_discount':data['min_discount'],'max_discount':data['max_discount'],'discountType':data['discountType'].upper().strip()}},upsert=False, multi = False)

        get_mongo_connection().Catalog.MasterData.update({'brand':data['brand'],'category_id':data['category_id']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data updated successfully"}

    except:

        return {0:"Data not updated."}

def getAllCategoryDiscount():

    data = []

    collection = get_mongo_connection().Catalog.CategoryDiscount

    cursor = collection.find()

    for val in cursor:

        data.append(val)

    return data

def __getBundledSkusfromSku(sku):

    masterData =  get_mongo_connection().Catalog.MasterData.find_one({"_id":sku},{"skuBundleId":1})

    if masterData is not None:

        return list(get_mongo_connection().Catalog.MasterData.find({"skuBundleId":masterData.get('skuBundleId')},{'_id':1,'skuBundleId':1}))

    else:

        return []

def addSchemeDetailsForSku(data):

    collection = get_mongo_connection().Catalog.SkuSchemeDetails

    result = collection.find_one({'skuBundleId':data['skuBundleId']})

    if result is None:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

    return {1:"BundleId info already present"}

def getAllSkuWiseSchemeDetails(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.SkuSchemeDetails

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def addSkuDiscountInfo(data):

    collection = get_mongo_connection().Catalog.SkuDiscountInfo

    cursor = collection.find_one({"skuBundleId":data['skuBundleId']})

    if cursor is not None:

        return {0:"BundleId information already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def getallSkuDiscountInfo(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.SkuDiscountInfo

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def updateSkuDiscount(data,_id):

    try:

        collection = get_mongo_connection().Catalog.SkuDiscountInfo

        collection.update({'_id':ObjectId(_id)},{"$set":{'min_discount':data['min_discount'],'max_discount':data['max_discount'],'discountType':data['discountType'].upper().strip()}},upsert=False, multi = False)

        get_mongo_connection().Catalog.MasterData.update({'_id':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data updated successfully"}

    except:

        return {0:"Data not updated."}

def addExceptionalNlc(data):

    collection = get_mongo_connection().Catalog.ExceptionalNlc

    cursor = collection.find_one({"skuBundleId":data['skuBundleId']})

    if cursor is not None:

        return {0:"BundleId information already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def getAllExceptionlNlcItems(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.ExceptionalNlc

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def getMerchantOrdersByUser(userId, page=1, window=50, searchMap={}):

    if searchMap is None:

        searchMap = {}

    if page==None:

        page = 1

    if window==None:

        window = 50

    result = {}

    skip = (page-1)*window

    if userId is not None:

        searchMap['userId'] = userId

    collection = get_mongo_connection().Dtr.merchantOrder

    cursor = collection.find(searchMap).sort("orderId",-1)

    total_count = cursor.count()

    pages = total_count/window + (0 if total_count%window==0 else 1)  

    print "total_count", total_count

    if total_count > skip:

        cursor = cursor.skip(skip).limit(window)

        orders = []

        for order in cursor:

            del(order["_id"])

            orders.append(order)

        result['data'] = orders

        result['window'] = window

        result['totalCount'] = total_count 

        result['currCount'] = cursor.count()

        result['totalPages'] = pages

        result['currPage'] = page    

        return result

    else:

        return result

def getRefunds(userId, page=1, window=10):

    if page==None:

        page = 1

    if window==None:

        window = 10

    result = {}

    skip = (page-1)*window

    con = get_mongo_connection()

    collection = con.Dtr.refund

    user = con.Dtr.user

    credited = user.find_one({"userId":userId})

    if credited:

        credited = credited.get("credited")

    else:

        credited=0

    cursor = collection.find({"userId":userId})

    total_count = cursor.count()

    pages = total_count/window + (0 if total_count%window==0 else 1)  

    print "total_count", total_count

    if total_count > skip:

        cursor = cursor.skip(skip).limit(window).sort([('batch',-1)])

        refunds = []

        for refund in cursor:

            del(refund["_id"])

            if (refund["type"]) == CREDIT_TYPE_REFUND:

                batchDetails = fetchCrmRefundByBatchId(refund["batch"])

                referenceNumber = batchDetails.get('reference_no')                

                refund["description"] = "Refunded against Saholic order #%s"%(referenceNumber)

            elif (refund["type"]) == CREDIT_TYPE_ADJUSTMENT:

                batchDetails = fetchCrmRefundByBatchId(refund["batch"])

                referenceNumber = batchDetails.get('reference_no')

                referenceStore = REVERSE_SOURCE_MAP.get(batchDetails.get('store'))

                refund["description"] = "Adjusted against %s order #%s"%(referenceStore,referenceNumber)

            elif (refund["type"]) == CREDIT_TYPE_OFFER:

                refund["description"] = "April Accessories Cashback Offer"                

            else:

                refund["description"] = ""

            refunds.append(refund)

        result['credited'] = credited

        result['data'] = refunds

        result['window'] = window

        result['totalCount'] = total_count 

        result['currCount'] = cursor.count()

        result['totalPages'] = pages

        result['currPage'] = page    

        return result

    else:

        return result

def getPendingRefunds(userId):

    print type(userId)

    result = get_mongo_connection().Dtr.merchantOrder\

        .aggregate([

                    {'$match':{'subOrders.cashBackStatus':CB_APPROVED, 'userId':userId}},

                    {'$unwind':"$subOrders"},

                    {'$match':{'subOrders.cashBackStatus':CB_APPROVED}},

                    { 

                     '$group':{

                               '_id':None,

                               'amount': { '$sum':'$subOrders.cashBackAmount'},

                               }

                     }

                ])['result']

    if len(result)>0:

        result = result[0]        

        result.pop("_id")

    else:

        result={}

        result['amount'] = 0.0

    result['nextCredit'] = datetime.strftime(next_weekday(datetime.now(), int(PythonPropertyReader.getConfig('CREDIT_DAY_OF_WEEK'))),"%Y-%m-%d %H:%M:%S")

    return result

def getPendingCashbacks(userId):

    result = get_mongo_connection().Dtr.merchantOrder\

        .aggregate([

                    {'$match':{'subOrders.cashBackStatus':CB_PENDING, 'userId':userId}},

                    {'$unwind':"$subOrders"},

                    {'$match':{'subOrders.cashBackStatus':CB_PENDING}},

                    { 

                     '$group':{

                               '_id':None,

                               'amount': { '$sum':'$subOrders.cashBackAmount'},

                               }

                     }

                ])['result']

    if len(result)>0:

        result = result[0]        

        result.pop("_id")

    else:

        result={}

        result['amount'] = 0.0

    return result

def __constructDummyDealObject(skuBundleId):

    temp =[]

    deals = get_mongo_connection().Catalog.Deals.find({"skuBundleId":skuBundleId,"$or":[{"showDeal":1}, {"prepaidDeal":1}]},{'_id':1,'category_id':1,'brand':1,'totalPoints':1,'bestSellerPoints':1,'nlcPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1,'subCategoryId':1,'netPriceAfterCashBack':1})

    for d in deals:

        d['persPoints'] = 0

        d['dealRankPoints'] = 0

        temp.append(d)

    return temp

def __populateCache(userId):

    try:

        if mc.get("featured_deals") is None:

            __populateFeaturedDeals()

        featuredDeals = mc.get("featured_deals")

    except Exception as e:

        print traceback.print_exc()

        featuredDeals = {3:{},5:{},6:{}}

    #Not taking accessories into account

    featuredDeals.pop(6,None)

    print featuredDeals

    fd_bundles = []

    for catMap in featuredDeals.itervalues():

        for fd_map in catMap.itervalues():

            if not fd_map.get('skuBundleId') in fd_bundles:

                fd_bundles.append(fd_map.get('skuBundleId'))

    print "Populating memcache for userId",userId

    outer_query = []

    outer_query.append({ "$or": [ { "showDeal": 1} , { "prepaidDeal": 1 } ] })

    query = {}

    query['$gt'] = -100

    outer_query.append({'totalPoints':query})

    outer_query.append({'category_id':{"$in":[3,5]}})

    brandPrefMap = {}

    pricePrefMap = {}

    actionsMap = {}

    try:

        brand_p = session.query(price_preferences).filter_by(user_id=userId).all()

        for x in brand_p:

            pricePrefMap[x.category_id] = [x.min_price,x.max_price]

        for x in session.query(brand_preferences).filter_by(user_id=userId).all():

            temp_map = {}

            if brandPrefMap.has_key((x.brand).strip().upper()):

                val = brandPrefMap.get((x.brand).strip().upper())

                temp_map[x.category_id] = 1 if x.status == 'show' else 0

                val.append(temp_map)

            else:

                temp = []

                temp_map[x.category_id] = 1 if x.status == 'show' else 0

                temp.append(temp_map)

                brandPrefMap[(x.brand).strip().upper()] = temp

        for x in session.query(user_actions).filter_by(user_id=userId).all():

            actionsMap[x.store_product_id] = 1 if x.action == 'like' else 0

    except:

        pass

    finally:

        session.close()

    all_deals = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query},{'_id':1,'category_id':1,'brand':1,'totalPoints':1,'bestSellerPoints':1,'nlcPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1, 'subCategoryId':1,'netPriceAfterCashBack':1}).sort([('totalPoints',pymongo.DESCENDING),('bestSellerPoints',pymongo.DESCENDING),('nlcPoints',pymongo.DESCENDING),('rank',pymongo.DESCENDING)]))

    mobile_deals = []

    tablet_deals = []

    #accessories_deals = []

    for deal in all_deals:

        try:

            fd_bundles.remove(deal.get('skuBundleId'))

        except:

            pass

        if actionsMap.get(deal['_id']) == 0:

            fav_weight =.25

        elif actionsMap.get(deal['_id']) == 1:

            fav_weight = 1.5

        else:

            fav_weight = 1

        if brandPrefMap.get(deal['brand'].strip().upper()) is not None:

            brand_weight = 1

            for brandInfo in brandPrefMap.get(deal['brand'].strip().upper()):

                if brandInfo.get(deal['category_id']) is not None:

                    if brandInfo.get(deal['category_id']) == 1:

                        brand_weight = 2.0

        else:

            brand_weight = 1

        if pricePrefMap.get(deal['category_id']) is not None:

            if deal['available_price'] >= pricePrefMap.get(deal['category_id'])[0] and deal['available_price'] <= pricePrefMap.get(deal['category_id'])[1]:

                asp_weight = 1.5

            elif  deal['available_price'] >= pricePrefMap.get(deal['category_id'])[0] - 0.5 * pricePrefMap.get(deal['category_id'])[0] and deal['available_price'] <= pricePrefMap.get(deal['category_id'])[1] + 0.5 * pricePrefMap.get(deal['category_id'])[1]:

                asp_weight = 1.2

            else:

                asp_weight = 1

        else:

            asp_weight = 1

        persPoints = deal['totalPoints'] * fav_weight * brand_weight * asp_weight

        deal['persPoints'] = persPoints

        if deal['category_id'] ==3:

            mobile_deals.append(deal)

        elif deal['category_id'] ==5:

            tablet_deals.append(deal)

        #elif deal['category_id'] ==6:

        #    accessories_deals.append(deal)

        else:

            continue

    mobile_deals = sorted(mobile_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    tablet_deals = sorted(tablet_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    #accessories_deals = sorted(accessories_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    if len(fd_bundles) > 0:

        print "There is still bundle in feature deals with points less than -100.Lets add info"

        for skuBundleId in fd_bundles:

            dummyDealObjList = __constructDummyDealObject(skuBundleId)

            for dummyDealObj in dummyDealObjList:

                if dummyDealObj['category_id'] ==3:

                    mobile_deals.append(dummyDealObj)

                elif dummyDealObj['category_id'] ==5:

                    tablet_deals.append(dummyDealObj)

                #elif dummyDealObj['category_id'] ==6:

                #    accessories_deals.append(dummyDealObj)

                else:

                    pass

    for super_category, categoryFeaturedDeals in featuredDeals.iteritems():

        for val in categoryFeaturedDeals.itervalues():

            dummyDealObjList = __constructDummyDealObject(val.get('skuBundleId'))

            for dummyDealObj in dummyDealObjList:

                if super_category == dummyDealObj['category_id']:

                    continue  

                if super_category ==3:

                    mobile_deals.append(dummyDealObj)

                elif super_category ==5:

                    tablet_deals.append(dummyDealObj)

                #elif super_category ==6:

                #    accessories_deals.append(dummyDealObj)

                else:

                    pass

    sortedMapMobiles = {}

    rankMapMobiles = {}

    rankMobiles = 0

    blockedRanksMobiles = []

    blockedSkuBundlesMobiles = []

    blockedInfoMobiles = {}

    featuredDealsMobiles = featuredDeals.get(3)

    for k, v in featuredDealsMobiles.iteritems():

        blockedRanksMobiles.append(k-1)

        blockedSkuBundlesMobiles.append(v.get('skuBundleId'))

    for sorted_deal in mobile_deals:

        while(True):

            if rankMobiles in blockedRanksMobiles:

                rankMobiles = rankMobiles +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesMobiles:

            if blockedInfoMobiles.has_key(sorted_deal['skuBundleId']):

                blockedInfoMobiles.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoMobiles[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapMobiles.get(sorted_deal['skuBundleId']) is None:

            sortedMapMobiles[sorted_deal['skuBundleId']] = {rankMobiles:[sorted_deal]}

            rankMapMobiles[rankMobiles] = (sortedMapMobiles[sorted_deal['skuBundleId']].values())[0]

            rankMobiles = rankMobiles +1

        else:

            for temp_list in sortedMapMobiles.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapMobiles[(sortedMapMobiles.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksMobiles:

        if blockedInfoMobiles.get((featuredDealsMobiles.get(rank+1)).get('skuBundleId')) is not None:

            rankMapMobiles[rank] = blockedInfoMobiles.get((featuredDealsMobiles.get(rank+1)).get('skuBundleId'))

    sortedMapTablets = {}

    rankMapTablets = {}

    rankTablets = 0

    blockedRanksTablets = []

    blockedSkuBundlesTablets = []

    blockedInfoTablets = {}

    featuredDealsTablets = featuredDeals.get(5)

    for k, v in featuredDealsTablets.iteritems():

        blockedRanksTablets.append(k-1)

        blockedSkuBundlesTablets.append(v.get('skuBundleId'))

    for sorted_deal in tablet_deals:

        while(True):

            if rankTablets in blockedRanksTablets:

                rankTablets = rankTablets +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesTablets:

            if blockedInfoTablets.has_key(sorted_deal['skuBundleId']):

                blockedInfoTablets.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoTablets[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapTablets.get(sorted_deal['skuBundleId']) is None:

            sortedMapTablets[sorted_deal['skuBundleId']] = {rankTablets:[sorted_deal]}

            rankMapTablets[rankTablets] = (sortedMapTablets[sorted_deal['skuBundleId']].values())[0]

            rankTablets = rankTablets +1

        else:

            for temp_list in sortedMapTablets.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapTablets[(sortedMapTablets.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksTablets:

        if blockedInfoTablets.get((featuredDealsTablets.get(rank+1)).get('skuBundleId')) is not None:

            rankMapTablets[rank] = blockedInfoTablets.get((featuredDealsTablets.get(rank+1)).get('skuBundleId'))

#     sortedMapAccessories = {}

#     rankMapAccessories = {}

#     rankAccessories = 0

#     

#     blockedRanksAccessories = []

#     blockedSkuBundlesAccessories = []

#     blockedInfoAccessories = {}

#     

#     featuredDealsAccessories = featuredDeals.get(6)

#     

#     for k, v in featuredDealsAccessories.iteritems():

#         blockedRanksAccessories.append(k-1)

#         blockedSkuBundlesAccessories.append(v.get('skuBundleId'))

#     

#     

#     for sorted_deal in accessories_deals:

#        

#         while(True):

#             if rankAccessories in blockedRanksAccessories:

#                 rankAccessories = rankAccessories +1

#             else:

#                 break

#              

#         if sorted_deal['skuBundleId'] in blockedSkuBundlesAccessories:

#             if blockedInfoAccessories.has_key(sorted_deal['skuBundleId']):

#                 blockedInfoAccessories.get(sorted_deal['skuBundleId']).append(sorted_deal)

#             else:

#                 blockedInfoAccessories[sorted_deal['skuBundleId']] = [sorted_deal]

#             continue

#          

#         if sortedMapAccessories.get(sorted_deal['skuBundleId']) is None:

#             sortedMapAccessories[sorted_deal['skuBundleId']] = {rankAccessories:[sorted_deal]}

#             rankMapAccessories[rankAccessories] = (sortedMapAccessories[sorted_deal['skuBundleId']].values())[0]

#             rankAccessories = rankAccessories +1

#         else:

#             for temp_list in sortedMapAccessories.get(sorted_deal['skuBundleId']).itervalues():

#                 temp_list.append(sorted_deal)

#             rankMapAccessories[(sortedMapAccessories.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

#     

#             

#     for rank in blockedRanksAccessories:

#         if blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId')) is not None:

#             rankMapAccessories[rank] = blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId'))

#     

#    specialOffer = None

#    promoOffer = get_mongo_connection().Catalog.PromoOffer.find_one({'user_id':userId})

#    try:

#        if promoOffer is not None:

#            offer = get_mongo_connection().Catalog.Promotions.find_one({'_id':promoOffer.get('offer_id')})

#            if offer is None:

#                raise

#            promo = Promotion(offer.get('_id'),offer.get('offer_name'),offer.get('offer_description') , offer.get('categories_applicable'), offer.get('sub_categories_not_applicable'),

#                     offer.get('startDate'), offer.get('endDate'), promoOffer.get('target1'), promoOffer.get('target1_cash_back_percetage'), promoOffer.get('target2'), promoOffer.get('target2_cash_back_percetage'), promoOffer.get('maxCashBack'),offer.get('url'),

#                     promoOffer.get('pending_order_value'), promoOffer.get('delivered_order_value'), promoOffer.get('last_run_timestamp'))

#    

#            specialOffer = promo

#    except:

#        traceback.print_exc()

    mem_cache_val = {3:mobile_deals, 5:tablet_deals,"3_rankMap": rankMapMobiles,"5_rankMap": rankMapTablets}

    mc.set(str(userId), mem_cache_val)

def __populateCacheForAccessories(userId):

    try:

        if mc.get("featured_deals") is None:

            __populateFeaturedDeals()

        featuredDeals = mc.get("featured_deals")

    except Exception as e:

        print traceback.print_exc()

        featuredDeals = {3:{},5:{},6:{}}

    #Not taking into account mobiles, tablets

    featuredDeals.pop(3,None)

    featuredDeals.pop(5,None)

    print "featuredDeals are ",featuredDeals

    fd_bundles = []

    for catMap in featuredDeals.itervalues():

        for fd_map in catMap.itervalues():

            if not fd_map.get('skuBundleId') in fd_bundles:

                fd_bundles.append(fd_map.get('skuBundleId'))

    print "Populating accessory memcache for userId",userId

    outer_query = []

    outer_query.append({ "$or": [ { "showDeal": 1} , { "prepaidDeal": 1 } ] })

    query = {}

    query['$gt'] = 0

    outer_query.append({'dealRankPoints':query})

    outer_query.append({'category_id':{"$in":[6]}})

    accessories_deals = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query},{'_id':1,'category_id':1,'brand':1,'bestSellerPoints':1,'nlcPoints':1,'totalPoints':1,'dealRankPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1, 'subCategoryId':1,'netPriceAfterCashBack':1}).sort([('dealRankPoints',pymongo.DESCENDING),('bestSellerPoints',pymongo.DESCENDING)]))

    actionsMap = {}

    actionRank = 3

    try:

        favourite_data = session.query(user_actions).filter_by(user_id=userId).order_by(asc(user_actions.created)).all()

        favourite_ids = [x.store_product_id for x in favourite_data]

        master_ids = list(get_mongo_connection().Catalog.MasterData.find({'_id':{"$in":favourite_ids},'category_id':6},{'_id':1}))

        accessory_ids = [x['_id'] for x in master_ids]

        for x in favourite_data:

            if x.store_product_id not in accessory_ids:

                continue

            action = 1 if x.action == 'like' else 0

            if action==1:

                actionsMap[x.store_product_id] = {'action':action,'rank':actionRank,'dealRankPoints':None}

                actionRank = actionRank+3

            else:

                actionsMap[x.store_product_id] = {'action':action,'rank':None,'dealRankPoints':None}

    except:

        traceback.print_exc()

        print "Exception in actionsMap creation"

    finally:    

        session.close()

    pseudo_count = 1

    inserted_count = 0

    try:

        if actionsMap:

            for k,v in actionsMap.iteritems():

                tmp_deal = get_mongo_connection().Catalog.Deals.find_one({'_id':k})

                if tmp_deal is None:

                    v['rank'] = None

                    continue

                if tmp_deal['category_id']!=6 and (tmp_deal['showDeal']!=1 or tmp_deal['prepaidDeal']!=1) :

                    v['rank'] = None

                    continue

                if v['action'] ==0:

                    leastDealRankPoints = list(get_mongo_connection().Catalog.Deals.find({'subCategoryId':tmp_deal.get('subCategoryId')}).sort([('dealRankPoints',pymongo.ASCENDING)]).limit(1))

                    if len(leastDealRankPoints) == 0 or leastDealRankPoints[0].get('dealRankPoints') is None:

                        v['rank'] = None

                        continue

                    v['dealRankPoints'] = leastDealRankPoints[0]['dealRankPoints'] - 1

                    v['rank'] = 999999

                else:

                    v['dealRankPoints'] = tmp_deal.get('dealRankPoints')

    except:

        print "Exception in action map"

        traceback.print_exc()         

    for deal in accessories_deals:

        try:

            fd_bundles.remove(deal.get('skuBundleId'))

        except:

            pass

        try:

            if actionsMap and actionsMap.get(deal['_id'])!=None:

                if actionsMap.get(deal['_id']).get('action') ==1 and actionsMap.get(deal['_id']).get('rank')!=None:

                    actionRank = actionsMap.get(deal['_id']).get('rank')

                    if actionRank < pseudo_count:

                        deal['dealRankPoints'] = accessories_deals[actionRank-1]['dealRankPoints'] +.5 + inserted_count

                        actionsMap.get(deal['_id'])['dealRankPoints'] = deal['dealRankPoints'] 

                        inserted_count = inserted_count+1

                elif actionsMap.get(deal['_id']).get('action') ==0 and actionsMap.get(deal['_id']).get('rank')!=None:

                    deal['dealRankPoints'] = actionsMap.get(deal['_id']).get('dealRankPoints')

                    actionsMap.get(deal['_id'])['dealRankPoints'] = deal['dealRankPoints']

                else:

                    pass

            pseudo_count = pseudo_count+1

        except:

            print "Exception while arranging dealRankPoints"

            traceback.print_exc()

    accessories_deals = sorted(accessories_deals, key = lambda x: (x['dealRankPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    if len(fd_bundles) > 0:

        print "There is still bundle in feature deals.Lets add info"

        for skuBundleId in fd_bundles:

            dummyDealObjList = __constructDummyDealObject(skuBundleId)

            for dummyDealObj in dummyDealObjList:

                if dummyDealObj['category_id'] ==6:

                    accessories_deals.append(dummyDealObj)

    for super_category, categoryFeaturedDeals in featuredDeals.iteritems():

        for val in categoryFeaturedDeals.itervalues():

            dummyDealObjList = __constructDummyDealObject(val.get('skuBundleId'))

            for dummyDealObj in dummyDealObjList:

                if super_category == dummyDealObj['category_id']:

                    continue  

                if super_category ==6:

                    accessories_deals.append(dummyDealObj)

    sortedMapAccessories = {}

    rankMapAccessories = {}

    rankAccessories = 0

    blockedRanksAccessories = []

    blockedSkuBundlesAccessories = []

    blockedInfoAccessories = {}

    featuredDealsAccessories = featuredDeals.get(6)

    for k, v in featuredDealsAccessories.iteritems():

        blockedRanksAccessories.append(k-1)

        blockedSkuBundlesAccessories.append(v.get('skuBundleId'))

    for sorted_deal in accessories_deals:

        while(True):

            if rankAccessories in blockedRanksAccessories:

                rankAccessories = rankAccessories +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesAccessories:

            if blockedInfoAccessories.has_key(sorted_deal['skuBundleId']):

                blockedInfoAccessories.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoAccessories[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapAccessories.get(sorted_deal['skuBundleId']) is None:

            sortedMapAccessories[sorted_deal['skuBundleId']] = {rankAccessories:[sorted_deal]}

            rankMapAccessories[rankAccessories] = (sortedMapAccessories[sorted_deal['skuBundleId']].values())[0]

            rankAccessories = rankAccessories +1

        else:

            for temp_list in sortedMapAccessories.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapAccessories[(sortedMapAccessories.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksAccessories:

        if blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId')) is not None:

            rankMapAccessories[rank] = blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId'))

    specialOffer = None

    promoOffer = get_mongo_connection().Catalog.PromoOffer.find_one({'user_id':userId})

    try:

        if promoOffer is not None:

            offer = get_mongo_connection().Catalog.Promotions.find_one({'_id':promoOffer.get('offer_id')})

            if offer is None:

                raise

            promo = Promotion(offer.get('_id'),offer.get('offer_name'),offer.get('offer_description') , offer.get('categories_applicable'), offer.get('sub_categories_not_applicable'),

                     offer.get('startDate'), offer.get('endDate'), promoOffer.get('target1'), promoOffer.get('target1_cash_back_percetage'), promoOffer.get('target2'), promoOffer.get('target2_cash_back_percetage'), promoOffer.get('maxCashBack'),offer.get('url'),

                     promoOffer.get('pending_order_value'), promoOffer.get('delivered_order_value'), promoOffer.get('last_run_timestamp'))

            specialOffer = promo

    except:

        traceback.print_exc()

    mem_cache_val = {6:accessories_deals,"6_rankMap": rankMapAccessories, "specialOffer":specialOffer }

    mc.set(str(userId)+'_acc', mem_cache_val)

def __populateFeaturedDeals():

    print "Populating featured deals....."

    featuredDealsMobiles = {}

    featuredDealsTablets = {}

    featuredDealsAccessories = {}

    activeFeaturedDeals = get_mongo_connection().Catalog.FeaturedDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})

    for activeFeaturedDeal in activeFeaturedDeals:

        deals = get_mongo_connection().Catalog.Deals.find({"skuBundleId":activeFeaturedDeal.get('skuBundleId'),"$or":[{"showDeal":1}, {"prepaidDeal":1}]})

        for deal in deals:

            if deal.get('available_price') <= activeFeaturedDeal.get('thresholdPrice'):

                for category_id, rank  in activeFeaturedDeal['otherInfo'].iteritems():

                    f_deal = {'skuBundleId':activeFeaturedDeal.get('skuBundleId'), 'thresholdPrice':activeFeaturedDeal.get('thresholdPrice'), 'rank':rank,'category_id':int(category_id),'totalPoints':activeFeaturedDeal.get('totalPoints')}

                    if f_deal['category_id'] == 3:

                        featuredDealsMobiles[rank] = f_deal

                    elif f_deal['category_id']==5:

                        featuredDealsTablets[rank] = f_deal

                    elif f_deal['category_id']==6:

                        featuredDealsAccessories[rank] = f_deal

                break

#                f_deal = {'skuBundleId':activeFeaturedDeal.get('skuBundleId'), 'thresholdPrice':activeFeaturedDeal.get('thresholdPrice'), 'rank':activeFeaturedDeal.get('rank'),'category_id':activeFeaturedDeal.get('category_id'),'totalPoints':activeFeaturedDeal.get('totalPoints')}

#                 if activeFeaturedDeal.get('category_id') == 3:

#                     featuredDealsMobiles[activeFeaturedDeal.get('rank')] = f_deal

#                 else:

#                     featuredDealsTablets[activeFeaturedDeal.get('rank')] = f_deal

#                 break

    mc.set("featured_deals", {3:featuredDealsMobiles,5:featuredDealsTablets,6:featuredDealsAccessories}, 600)

def __populateFeaturedDealsForDealObject():

    print "Caching deal objects...."

    featuredDealObjectMobiles = {}

    featuredDealObjectTablets = {}

    featuredDealObjectAccessories = {}

    dealObjects = get_mongo_connection().Catalog.FeaturedDealsDealObject.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})

    for dealObject in dealObjects:

        d_object = list(get_mongo_connection().Catalog.DealObject.find({'_id':dealObject['_id']}))

        if len(d_object) == 0:

            continue

        for category_id, rank  in dealObject['otherInfo'].iteritems():

            d_object[0]['dealObject'] = 1

            f_deal = {'rank':rank,'category_id':int(category_id),'object':d_object[0],'offer_id':dealObject.get('offer_id')}

            if f_deal['category_id'] == 3:

                featuredDealObjectMobiles[rank] = f_deal

            elif f_deal['category_id']==5:

                featuredDealObjectTablets[rank] = f_deal

            elif f_deal['category_id']==6:

                featuredDealObjectAccessories[rank] = f_deal

    mc.set("featured_deals_deal_object", {3:featuredDealObjectMobiles,5:featuredDealObjectTablets,6:featuredDealObjectAccessories}, 600)

    print mc.get('featured_deals_deal_object')

#introducing online/deals with fofo flag

def getNewDeals(userId, category_id, offset, limit, sort, direction, filterData=None, source=None, tag_ids=None):

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    if source is not None:

        if source=="online":

            sourceToFilter =   [str(i) for i in ONLINE_DEAL_SOURCE_MAP.keys()]  #filter categores on basis of source and tags

        #For saholic specific deals

        elif source=="deals":

            sourceToFilter = ['4']

        #we should override sourceFilterString

        sourceFiterString = "sourceFilter:" + "^".join(sourceToFilter) 

        if filterData is None:

            filterData = sourceFiterString

        else:

            a = filterData.split('sourceFilter:')

            if len(a) > 1:

                filterData =  a[0] + sourceFiterString

                b = a[1].split("|")

                if len(b) > 1:

                    filterData = "|".join([filterData] + b[1:])

            else:

                filterData = a + "|" + sourceFiterString

    dealsListMap = []

    user_specific_deals = mc.get(str(userId))

    if user_specific_deals is None:

        __populateCache(userId)

        user_specific_deals = mc.get(str(userId))

    else:

        print "Getting user deals from cache"

    category_specific_deals = user_specific_deals.get(category_id)

    insert_featured_deals = False

    if sort is None or direction is None:

        insert_featured_deals = True

        rankMap = user_specific_deals.get(str(category_id)+"_rankMap")

    else:

        if sort == "bestSellerPoints":

            sorted_deals = sorted(category_specific_deals, key = lambda x: (x['bestSellerPoints'], x['rank'], x['nlcPoints']),reverse=True)

        else:

            if direction == -1:

                rev = True

            else:

                rev = False

            sorted_deals = sorted(category_specific_deals, key = lambda x: (x['available_price']),reverse=rev)

    filtered_deals = []

    dataExist = True