Subversion Repositories SmartDukaan

from bson.objectid import ObjectId

from datetime import datetime, timedelta

from dtr.config import PythonPropertyReader

from dtr.dao import FeaturedDeals, AppTransactions, AppOfferObj, Promotion

from dtr.storage import DataService

from dtr.storage.DataService import price_preferences, brand_preferences, \

    user_actions, Brands, app_offers, approved_app_transactions, user_app_cashbacks, \

    user_app_installs, appmasters, notification_campaigns

from dtr.storage.MemCache import MemCache

from dtr.utils.MailSender import Email

from dtr.utils.utils import to_java_date, CB_PENDING, CB_APPROVED, CB_INIT, \

    to_py_date, CB_REJECTED, SUB_CATEGORY_MAP

from sqlalchemy.sql.expression import func, func, or_, desc, asc, case

from elixir import *

from operator import itemgetter

from pymongo.command_cursor import CommandCursor

import pymongo

import random

import re

import time

import traceback

from itertools import groupby

import urllib

import urllib2

import json

import collections

try:

    import ordereddict

except:

    pass

import bson.son as son

import math

con = None

es = None

try:

    from elasticsearch import Elasticsearch

except:

    pass

from shop2020.config.client.ConfigClient import ConfigClient

DataService.initialize(db_hostname="localhost")

mc = MemCache("127.0.0.1")

config_client = ConfigClient()

elastic_search_host = config_client.get_property('elastic_search_host')

elastic_search_port = config_client.get_property('elastic_search_port')

SOURCE_MAP = {1:'AMAZON',2:'FLIPKART',3:'SNAPDEAL',4:'SAHOLIC',5:"SHOPCLUES.COM",6:"PAYTM.COM",7:"HOMESHOP18.COM"}

COLLECTION_MAP = {

                  'ExceptionalNlc':'skuBundleId',

                  'SkuDealerPrices':'skuBundleId',

                  'SkuDiscountInfo':'skuBundleId',

                  'SkuSchemeDetails':'skuBundleId',

                  'DealPoints':'skuBundleId',

                  'FeaturedDeals': 'skuBundleId'

                  }

def get_elastic_search_connection():

    global es

    if es is None:

        print "Establishing connection with elastic search %s host and port %s"%(elastic_search_host,elastic_search_port)

        es = Elasticsearch([{'host': elastic_search_host, 'port': elastic_search_port}])

        return es

    return es

def get_mongo_connection(host='localhost', port=27017):

    global con

    if con is None:

        print "Establishing connection %s host and port %d" %(host,port)

        try:

            con = pymongo.MongoClient(host, port)

        except Exception, e:

            print e

            return None

    return con

def populateCashBack():

    print "Populating cashback"

    cashBackMap = {}

    itemCashBackMap = {}

    cashBack = list(get_mongo_connection().Catalog.CategoryCashBack.find())

    for row in cashBack:

        temp_map = {}

        temp_list = []

        if cashBackMap.has_key(row['source_id']):

            arr = cashBackMap.get(row['source_id'])

            for val in arr:

                temp_list.append(val)

            temp_map[row['category_id']] = row

            temp_list.append(temp_map)

            cashBackMap[row['source_id']] = temp_list 

        else:

            temp_map[row['category_id']] = row

            temp_list.append(temp_map)

            cashBackMap[row['source_id']] = temp_list

    itemCashBack = list(get_mongo_connection().Catalog.ItemCashBack.find())

    for row in itemCashBack:

        if not itemCashBackMap.has_key(row['skuId']):

            itemCashBackMap[row['skuId']] = row

    mc.set("item_cash_back", itemCashBackMap, 24 * 60 * 60)

    mc.set("category_cash_back", cashBackMap, 24 * 60 * 60)

def addCategoryDiscount(data):

    collection = get_mongo_connection().Catalog.CategoryDiscount

    query = []

    data['brand'] = data['brand'].strip().upper()

    data['discountType'] = data['discountType'].upper().strip()

    query.append({"brand":data['brand']})

    query.append({"category_id":data['category_id']})

    r = collection.find({"$and":query})

    if r.count() > 0:

        return {0:"Brand & Category info already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'brand':data['brand'],'category_id':data['category_id']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def updateCategoryDiscount(data,_id):

    try:

        collection = get_mongo_connection().Catalog.CategoryDiscount

        collection.update({'_id':ObjectId(_id)},{"$set":{'min_discount':data['min_discount'],'max_discount':data['max_discount'],'discountType':data['discountType'].upper().strip()}},upsert=False, multi = False)

        get_mongo_connection().Catalog.MasterData.update({'brand':data['brand'],'category_id':data['category_id']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data updated successfully"}

    except:

        return {0:"Data not updated."}

def getAllCategoryDiscount():

    data = []

    collection = get_mongo_connection().Catalog.CategoryDiscount

    cursor = collection.find()

    for val in cursor:

        data.append(val)

    return data

def __getBundledSkusfromSku(sku):

    masterData =  get_mongo_connection().Catalog.MasterData.find_one({"_id":sku},{"skuBundleId":1})

    if masterData is not None:

        return list(get_mongo_connection().Catalog.MasterData.find({"skuBundleId":masterData.get('skuBundleId')},{'_id':1,'skuBundleId':1}))

    else:

        return []

def addSchemeDetailsForSku(data):

    collection = get_mongo_connection().Catalog.SkuSchemeDetails

    result = collection.find_one({'skuBundleId':data['skuBundleId']})

    if result is None:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

    return {1:"BundleId info already present"}

def getAllSkuWiseSchemeDetails(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.SkuSchemeDetails

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def addSkuDiscountInfo(data):

    collection = get_mongo_connection().Catalog.SkuDiscountInfo

    cursor = collection.find_one({"skuBundleId":data['skuBundleId']})

    if cursor is not None:

        return {0:"BundleId information already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def getallSkuDiscountInfo(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.SkuDiscountInfo

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def updateSkuDiscount(data,_id):

    try:

        collection = get_mongo_connection().Catalog.SkuDiscountInfo

        collection.update({'_id':ObjectId(_id)},{"$set":{'min_discount':data['min_discount'],'max_discount':data['max_discount'],'discountType':data['discountType'].upper().strip()}},upsert=False, multi = False)

        get_mongo_connection().Catalog.MasterData.update({'_id':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data updated successfully"}

    except:

        return {0:"Data not updated."}

def addExceptionalNlc(data):

    collection = get_mongo_connection().Catalog.ExceptionalNlc

    cursor = collection.find_one({"skuBundleId":data['skuBundleId']})

    if cursor is not None:

        return {0:"BundleId information already present."}

    else:

        collection.insert(data)

        get_mongo_connection().Catalog.MasterData.update({'skuBundleId':data['skuBundleId']},{"$set":{'updatedOn':to_java_date(datetime.now())}},multi=True)

        return {1:"Data added successfully"}

def getAllExceptionlNlcItems(offset, limit):

    data = []

    collection = get_mongo_connection().Catalog.ExceptionalNlc

    cursor = collection.find().skip(offset).limit(limit)

    for val in cursor:

        master = get_mongo_connection().Catalog.MasterData.find_one({'skuBundleId':val['skuBundleId']})

        if master is not None:

            val['brand'] = master['brand']

            val['source_product_name'] = master['source_product_name']

        else:

            val['brand'] = ""

            val['source_product_name'] = ""

        data.append(val)

    return data

def getMerchantOrdersByUser(userId, page=1, window=50, searchMap={}):

    if searchMap is None:

        searchMap = {}

    if page==None:

        page = 1

    if window==None:

        window = 50

    result = {}

    skip = (page-1)*window

    if userId is not None:

        searchMap['userId'] = userId

    collection = get_mongo_connection().Dtr.merchantOrder

    cursor = collection.find(searchMap).sort("orderId",-1)

    total_count = cursor.count()

    pages = total_count/window + (0 if total_count%window==0 else 1)  

    print "total_count", total_count

    if total_count > skip:

        cursor = cursor.skip(skip).limit(window)

        orders = []

        for order in cursor:

            del(order["_id"])

            orders.append(order)

        result['data'] = orders

        result['window'] = window

        result['totalCount'] = total_count 

        result['currCount'] = cursor.count()

        result['totalPages'] = pages

        result['currPage'] = page    

        return result

    else:

        return result

def getRefunds(userId, page=1, window=10):

    if page==None:

        page = 1

    if window==None:

        window = 10

    result = {}

    skip = (page-1)*window

    con = get_mongo_connection()

    collection = con.Dtr.refund

    user = con.Dtr.user

    credited = user.find_one({"userId":userId})

    if credited:

        credited = credited.get("credited")

    else:

        credited=0

    cursor = collection.find({"userId":userId})

    total_count = cursor.count()

    pages = total_count/window + (0 if total_count%window==0 else 1)  

    print "total_count", total_count

    if total_count > skip:

        cursor = cursor.skip(skip).limit(window).sort([('batch',-1)])

        refunds = []

        for refund in cursor:

            del(refund["_id"])

            refunds.append(refund)

        result['credited'] = credited

        result['data'] = refunds

        result['window'] = window

        result['totalCount'] = total_count 

        result['currCount'] = cursor.count()

        result['totalPages'] = pages

        result['currPage'] = page    

        return result

    else:

        return result

def getPendingRefunds(userId):

    print type(userId)

    result = get_mongo_connection().Dtr.merchantOrder\

        .aggregate([

                    {'$match':{'subOrders.cashBackStatus':CB_APPROVED, 'userId':userId}},

                    {'$unwind':"$subOrders"},

                    {'$match':{'subOrders.cashBackStatus':CB_APPROVED}},

                    { 

                     '$group':{

                               '_id':None,

                               'amount': { '$sum':'$subOrders.cashBackAmount'},

                               }

                     }

                ])['result']

    if len(result)>0:

        result = result[0]        

        result.pop("_id")

    else:

        result={}

        result['amount'] = 0.0

    result['nextCredit'] = datetime.strftime(next_weekday(datetime.now(), int(PythonPropertyReader.getConfig('CREDIT_DAY_OF_WEEK'))),"%Y-%m-%d %H:%M:%S")

    return result

def getPendingCashbacks(userId):

    result = get_mongo_connection().Dtr.merchantOrder\

        .aggregate([

                    {'$match':{'subOrders.cashBackStatus':CB_PENDING, 'userId':userId}},

                    {'$unwind':"$subOrders"},

                    {'$match':{'subOrders.cashBackStatus':CB_PENDING}},

                    { 

                     '$group':{

                               '_id':None,

                               'amount': { '$sum':'$subOrders.cashBackAmount'},

                               }

                     }

                ])['result']

    if len(result)>0:

        result = result[0]        

        result.pop("_id")

    else:

        result={}

        result['amount'] = 0.0

    return result

def __constructDummyDealObject(skuBundleId):

    temp =[]

    deals = get_mongo_connection().Catalog.Deals.find({"skuBundleId":skuBundleId,"$or":[{"showDeal":1}, {"prepaidDeal":1}]},{'_id':1,'category_id':1,'brand':1,'totalPoints':1,'bestSellerPoints':1,'nlcPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1,'subCategoryId':1,'netPriceAfterCashBack':1})

    for d in deals:

        d['persPoints'] = 0

        temp.append(d)

    return temp

def __populateCache(userId):

    try:

        if mc.get("featured_deals") is None:

            __populateFeaturedDeals()

        featuredDeals = mc.get("featured_deals")

    except Exception as e:

        print traceback.print_exc()

        featuredDeals = {3:{},5:{},6:{}}

    fd_bundles = []

    for catMap in featuredDeals.itervalues():

        for fd_map in catMap.itervalues():

            if not fd_map.get('skuBundleId') in fd_bundles:

                fd_bundles.append(fd_map.get('skuBundleId'))

    print "Populating memcache for userId",userId

    outer_query = []

    outer_query.append({ "$or": [ { "showDeal": 1} , { "prepaidDeal": 1 } ] })

    query = {}

    query['$gt'] = -100

    outer_query.append({'totalPoints':query})

    brandPrefMap = {}

    pricePrefMap = {}

    actionsMap = {}

    brand_p = session.query(price_preferences).filter_by(user_id=userId).all()

    for x in brand_p:

        pricePrefMap[x.category_id] = [x.min_price,x.max_price]

    for x in session.query(brand_preferences).filter_by(user_id=userId).all():

        temp_map = {}

        if brandPrefMap.has_key((x.brand).strip().upper()):

            val = brandPrefMap.get((x.brand).strip().upper())

            temp_map[x.category_id] = 1 if x.status == 'show' else 0

            val.append(temp_map)

        else:

            temp = []

            temp_map[x.category_id] = 1 if x.status == 'show' else 0

            temp.append(temp_map)

            brandPrefMap[(x.brand).strip().upper()] = temp

    for x in session.query(user_actions).filter_by(user_id=userId).all():

        actionsMap[x.store_product_id] = 1 if x.action == 'like' else 0

    all_deals = list(get_mongo_connection().Catalog.Deals.find({"$and":outer_query},{'_id':1,'category_id':1,'brand':1,'totalPoints':1,'bestSellerPoints':1,'nlcPoints':1,'rank':1,'available_price':1,'dealType':1,'source_id':1,'brand_id':1,'skuBundleId':1,'dp':1, 'showDp':1, 'gross_price':1, 'subCategoryId':1,'netPriceAfterCashBack':1}).sort([('totalPoints',pymongo.DESCENDING),('bestSellerPoints',pymongo.DESCENDING),('nlcPoints',pymongo.DESCENDING),('rank',pymongo.DESCENDING)]))

    mobile_deals = []

    tablet_deals = []

    accessories_deals = []

    for deal in all_deals:

        try:

            fd_bundles.remove(deal.get('skuBundleId'))

        except:

            pass

        if actionsMap.get(deal['_id']) == 0:

            fav_weight =.25

        elif actionsMap.get(deal['_id']) == 1:

            fav_weight = 1.5

        else:

            fav_weight = 1

        if brandPrefMap.get(deal['brand'].strip().upper()) is not None:

            brand_weight = 1

            for brandInfo in brandPrefMap.get(deal['brand'].strip().upper()):

                if brandInfo.get(deal['category_id']) is not None:

                    if brandInfo.get(deal['category_id']) == 1:

                        brand_weight = 2.0

        else:

            brand_weight = 1

        if pricePrefMap.get(deal['category_id']) is not None:

            if deal['available_price'] >= pricePrefMap.get(deal['category_id'])[0] and deal['available_price'] <= pricePrefMap.get(deal['category_id'])[1]:

                asp_weight = 1.5

            elif  deal['available_price'] >= pricePrefMap.get(deal['category_id'])[0] - 0.5 * pricePrefMap.get(deal['category_id'])[0] and deal['available_price'] <= pricePrefMap.get(deal['category_id'])[1] + 0.5 * pricePrefMap.get(deal['category_id'])[1]:

                asp_weight = 1.2

            else:

                asp_weight = 1

        else:

            asp_weight = 1

        persPoints = deal['totalPoints'] * fav_weight * brand_weight * asp_weight

        deal['persPoints'] = persPoints

        if deal['category_id'] ==3:

            mobile_deals.append(deal)

        elif deal['category_id'] ==5:

            tablet_deals.append(deal)

        elif deal['category_id'] ==6:

            accessories_deals.append(deal)

        else:

            continue

    session.close()

    mobile_deals = sorted(mobile_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    tablet_deals = sorted(tablet_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    accessories_deals = sorted(accessories_deals, key = lambda x: (x['persPoints'],x['totalPoints'],x['bestSellerPoints'], x['nlcPoints'], x['rank']),reverse=True)

    if len(fd_bundles) > 0:

        print "There is still bundle in feature deals with points less than -100.Lets add info"

        for skuBundleId in fd_bundles:

            dummyDealObjList = __constructDummyDealObject(skuBundleId)

            for dummyDealObj in dummyDealObjList:

                if dummyDealObj['category_id'] ==3:

                    mobile_deals.append(dummyDealObj)

                elif dummyDealObj['category_id'] ==5:

                    tablet_deals.append(dummyDealObj)

                elif dummyDealObj['category_id'] ==6:

                    accessories_deals.append(dummyDealObj)

                else:

                    pass

    for super_category, categoryFeaturedDeals in featuredDeals.iteritems():

        for val in categoryFeaturedDeals.itervalues():

            dummyDealObjList = __constructDummyDealObject(val.get('skuBundleId'))

            for dummyDealObj in dummyDealObjList:

                if super_category == dummyDealObj['category_id']:

                    continue  

                if super_category ==3:

                    mobile_deals.append(dummyDealObj)

                elif super_category ==5:

                    tablet_deals.append(dummyDealObj)

                elif super_category ==6:

                    accessories_deals.append(dummyDealObj)

                else:

                    pass

    sortedMapMobiles = {}

    rankMapMobiles = {}

    rankMobiles = 0

    blockedRanksMobiles = []

    blockedSkuBundlesMobiles = []

    blockedInfoMobiles = {}

    featuredDealsMobiles = featuredDeals.get(3)

    for k, v in featuredDealsMobiles.iteritems():

        blockedRanksMobiles.append(k-1)

        blockedSkuBundlesMobiles.append(v.get('skuBundleId'))

    for sorted_deal in mobile_deals:

        while(True):

            if rankMobiles in blockedRanksMobiles:

                rankMobiles = rankMobiles +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesMobiles:

            if blockedInfoMobiles.has_key(sorted_deal['skuBundleId']):

                blockedInfoMobiles.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoMobiles[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapMobiles.get(sorted_deal['skuBundleId']) is None:

            sortedMapMobiles[sorted_deal['skuBundleId']] = {rankMobiles:[sorted_deal]}

            rankMapMobiles[rankMobiles] = (sortedMapMobiles[sorted_deal['skuBundleId']].values())[0]

            rankMobiles = rankMobiles +1

        else:

            for temp_list in sortedMapMobiles.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapMobiles[(sortedMapMobiles.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksMobiles:

        if blockedInfoMobiles.get((featuredDealsMobiles.get(rank+1)).get('skuBundleId')) is not None:

            rankMapMobiles[rank] = blockedInfoMobiles.get((featuredDealsMobiles.get(rank+1)).get('skuBundleId'))

    sortedMapTablets = {}

    rankMapTablets = {}

    rankTablets = 0

    blockedRanksTablets = []

    blockedSkuBundlesTablets = []

    blockedInfoTablets = {}

    featuredDealsTablets = featuredDeals.get(5)

    for k, v in featuredDealsTablets.iteritems():

        blockedRanksTablets.append(k-1)

        blockedSkuBundlesTablets.append(v.get('skuBundleId'))

    for sorted_deal in tablet_deals:

        while(True):

            if rankTablets in blockedRanksTablets:

                rankTablets = rankTablets +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesTablets:

            if blockedInfoTablets.has_key(sorted_deal['skuBundleId']):

                blockedInfoTablets.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoTablets[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapTablets.get(sorted_deal['skuBundleId']) is None:

            sortedMapTablets[sorted_deal['skuBundleId']] = {rankTablets:[sorted_deal]}

            rankMapTablets[rankTablets] = (sortedMapTablets[sorted_deal['skuBundleId']].values())[0]

            rankTablets = rankTablets +1

        else:

            for temp_list in sortedMapTablets.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapTablets[(sortedMapTablets.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksTablets:

        if blockedInfoTablets.get((featuredDealsTablets.get(rank+1)).get('skuBundleId')) is not None:

            rankMapTablets[rank] = blockedInfoTablets.get((featuredDealsTablets.get(rank+1)).get('skuBundleId'))

    sortedMapAccessories = {}

    rankMapAccessories = {}

    rankAccessories = 0

    blockedRanksAccessories = []

    blockedSkuBundlesAccessories = []

    blockedInfoAccessories = {}

    featuredDealsAccessories = featuredDeals.get(6)

    for k, v in featuredDealsAccessories.iteritems():

        blockedRanksAccessories.append(k-1)

        blockedSkuBundlesAccessories.append(v.get('skuBundleId'))

    for sorted_deal in accessories_deals:

        while(True):

            if rankAccessories in blockedRanksAccessories:

                rankAccessories = rankAccessories +1

            else:

                break

        if sorted_deal['skuBundleId'] in blockedSkuBundlesAccessories:

            if blockedInfoAccessories.has_key(sorted_deal['skuBundleId']):

                blockedInfoAccessories.get(sorted_deal['skuBundleId']).append(sorted_deal)

            else:

                blockedInfoAccessories[sorted_deal['skuBundleId']] = [sorted_deal]

            continue

        if sortedMapAccessories.get(sorted_deal['skuBundleId']) is None:

            sortedMapAccessories[sorted_deal['skuBundleId']] = {rankAccessories:[sorted_deal]}

            rankMapAccessories[rankAccessories] = (sortedMapAccessories[sorted_deal['skuBundleId']].values())[0]

            rankAccessories = rankAccessories +1

        else:

            for temp_list in sortedMapAccessories.get(sorted_deal['skuBundleId']).itervalues():

                temp_list.append(sorted_deal)

            rankMapAccessories[(sortedMapAccessories.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

    for rank in blockedRanksAccessories:

        if blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId')) is not None:

            rankMapAccessories[rank] = blockedInfoAccessories.get((featuredDealsAccessories.get(rank+1)).get('skuBundleId'))

    specialOffer = None

    promoOffer = get_mongo_connection().Catalog.PromoOffer.find_one({'user_id':userId})

    try:

        if promoOffer is not None:

            offer = get_mongo_connection().Catalog.Promotions.find_one({'_id':promoOffer.get('offer_id')})

            if offer is None:

                raise

            promo = Promotion(offer.get('offer_id'),offer.get('offer_name'),offer.get('offer_description') , offer.get('categories_applicable'), offer.get('sub_categories_not_applicable'),

                     offer.get('startDate'), offer.get('endDate'), promoOffer.get('target1'), promoOffer.get('target1_cash_back_percetage'), promoOffer.get('target2'), promoOffer.get('target2_cash_back_percetage'), promoOffer.get('maxCashBack'),offer.get('url'),

                     promoOffer.get('pending_order_value'), promoOffer.get('delivered_order_value'))

            specialOffer = promo

    except:

        pass

    mem_cache_val = {3:mobile_deals, 5:tablet_deals, 6: accessories_deals,"3_rankMap": rankMapMobiles,"5_rankMap": rankMapTablets,"6_rankMap": rankMapAccessories, "specialOffer":specialOffer }

    mc.set(str(userId), mem_cache_val)

def __populateFeaturedDeals():

    print "Populating featured deals....."

    featuredDealsMobiles = {}

    featuredDealsTablets = {}

    featuredDealsAccessories = {}

    activeFeaturedDeals = get_mongo_connection().Catalog.FeaturedDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})

    for activeFeaturedDeal in activeFeaturedDeals:

        deals = get_mongo_connection().Catalog.Deals.find({"skuBundleId":activeFeaturedDeal.get('skuBundleId'),"$or":[{"showDeal":1}, {"prepaidDeal":1}]})

        for deal in deals:

            if deal.get('available_price') <= activeFeaturedDeal.get('thresholdPrice'):

                for category_id, rank  in activeFeaturedDeal['otherInfo'].iteritems():

                    f_deal = {'skuBundleId':activeFeaturedDeal.get('skuBundleId'), 'thresholdPrice':activeFeaturedDeal.get('thresholdPrice'), 'rank':rank,'category_id':int(category_id),'totalPoints':activeFeaturedDeal.get('totalPoints')}

                    if f_deal['category_id'] == 3:

                        featuredDealsMobiles[rank] = f_deal

                    elif f_deal['category_id']==5:

                        featuredDealsTablets[rank] = f_deal

                    elif f_deal['category_id']==6:

                        featuredDealsAccessories[rank] = f_deal

                break

#                f_deal = {'skuBundleId':activeFeaturedDeal.get('skuBundleId'), 'thresholdPrice':activeFeaturedDeal.get('thresholdPrice'), 'rank':activeFeaturedDeal.get('rank'),'category_id':activeFeaturedDeal.get('category_id'),'totalPoints':activeFeaturedDeal.get('totalPoints')}

#                 if activeFeaturedDeal.get('category_id') == 3:

#                     featuredDealsMobiles[activeFeaturedDeal.get('rank')] = f_deal

#                 else:

#                     featuredDealsTablets[activeFeaturedDeal.get('rank')] = f_deal

#                 break

    mc.set("featured_deals", {3:featuredDealsMobiles,5:featuredDealsTablets,6:featuredDealsAccessories}, 600)

def __populateFeaturedDealsForDealObject():

    print "Caching deal objects...."

    featuredDealObjectMobiles = {}

    featuredDealObjectTablets = {}

    featuredDealObjectAccessories = {}

    dealObjects = get_mongo_connection().Catalog.FeaturedDealsDealObject.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}})

    for dealObject in dealObjects:

        d_object = list(get_mongo_connection().Catalog.DealObject.find({'_id':dealObject['_id']}))

        if len(d_object) == 0:

            continue

        for category_id, rank  in dealObject['otherInfo'].iteritems():

            d_object[0]['dealObject'] = 1

            f_deal = {'rank':rank,'category_id':int(category_id),'object':d_object[0],'offer_id':dealObject.get('offer_id')}

            if f_deal['category_id'] == 3:

                featuredDealObjectMobiles[rank] = f_deal

            elif f_deal['category_id']==5:

                featuredDealObjectTablets[rank] = f_deal

            elif f_deal['category_id']==6:

                featuredDealObjectAccessories[rank] = f_deal

    mc.set("featured_deals_deal_object", {3:featuredDealObjectMobiles,5:featuredDealObjectTablets,6:featuredDealObjectAccessories}, 600)

    print mc.get('featured_deals_deal_object')

def getNewDeals(userId, category_id, offset, limit, sort, direction, filterData=None):

    if mc.get("category_cash_back") is None or not bool(mc.get("category_cash_back")):

        populateCashBack()

    dealsListMap = []

    user_specific_deals = mc.get(str(userId))

    if user_specific_deals is None:

        __populateCache(userId)

        user_specific_deals = mc.get(str(userId))

    else:

        print "Getting user deals from cache"

    category_specific_deals = user_specific_deals.get(category_id)

    insert_featured_deals = False

    if sort is None or direction is None:

        insert_featured_deals = True

        rankMap = user_specific_deals.get(str(category_id)+"_rankMap")

    else:

        if sort == "bestSellerPoints":

            sorted_deals = sorted(category_specific_deals, key = lambda x: (x['bestSellerPoints'], x['rank'], x['nlcPoints']),reverse=True)

        else:

            if direction == -1:

                rev = True

            else:

                rev = False

            sorted_deals = sorted(category_specific_deals, key = lambda x: (x['available_price']),reverse=rev)

    filtered_deals = []

    dataExist = True

    if filterData is not None:

        try:

            filtered_deals = filterDeals(category_specific_deals, filterData)

            if len(filtered_deals)==0:

                dataExist = False

        except:

            traceback.print_exc()

    if dataExist:

        if not insert_featured_deals:

            sortedMap = {}

            rankMap = {}

            rank = 0

            for sorted_deal in sorted_deals:

                if len(filtered_deals) > 0 and  sorted_deal['skuBundleId'] not in filtered_deals:

                    continue

                if sortedMap.get(sorted_deal['skuBundleId']) is None:

                    sortedMap[sorted_deal['skuBundleId']] = {rank:[sorted_deal]}

                    rankMap[rank] = (sortedMap[sorted_deal['skuBundleId']].values())[0]

                    rank = rank +1

                else:

                    for temp_list in sortedMap.get(sorted_deal['skuBundleId']).itervalues():

                        temp_list.append(sorted_deal)

                    rankMap[(sortedMap.get(sorted_deal['skuBundleId']).keys())[0]] = temp_list

        else:

            filterMap = {}

            rank = 0

            if len(filtered_deals) > 0:

                try:

                    od = collections.OrderedDict(sorted(rankMap.items()))

                except:

                    od = ordereddict.OrderedDict(sorted(rankMap.items()))

                for k, v in od.iteritems():

                    if v[0].get('skuBundleId') in filtered_deals:

                        filterMap[rank] = v

                        rank =rank+1

                rankMap = filterMap

        rankCounter = offset

        if filterData is None:

            if mc.get("featured_deals_deal_object") is None or not bool(mc.get("featured_deals_deal_object")):

                try:

                    __populateFeaturedDealsForDealObject()

                    fd_dealObject = mc.get("featured_deals_deal_object")

                except:

                    fd_dealObject = {3:{},5:{},6:{}}

            else:

                fd_dealObject = mc.get("featured_deals_deal_object")

    else:

        rankMap = {}

    specialOffer = user_specific_deals.get('specialOffer')

    for dealList in [rankMap.get(k, []) for k in range(offset, offset+limit)]:

        if filterData is None:

            while(True):

                rankCounter = rankCounter+1

                if (fd_dealObject.get(category_id).get(rankCounter)) is not None:

                    if (fd_dealObject.get(category_id).get(rankCounter).get('offer_id') !=0):

                        #Check whether user can see offer

                        if specialOffer is None or specialOffer.endDate > to_java_date(datetime.now()):

                            continue

                    deal_object_temp = []

                    deal_object_temp.append(fd_dealObject.get(category_id).get(rankCounter).get('object'))

                    dealsListMap.append(deal_object_temp)

                else:

                    break

        temp = []

        if dealList is None or len(dealList)==0:

            continue

        for d in dealList:

            item = list(get_mongo_connection().Catalog.MasterData.find({'_id':d['_id']}))

            if len(item) ==0:

                continue

            item[0]['persPoints'] = d['persPoints']

            if d['dealType'] == 1 and d['source_id'] ==1:

                try:

                    manualDeal = list(get_mongo_connection().Catalog.ManualDeals.find({'sku':d['_id'],'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())}}))

                    item[0]['marketPlaceUrl'] = manualDeal[0]['dealUrl'].strip()

                except:

                    pass

            elif d['source_id'] ==3:

                item[0]['marketPlaceUrl'] = item[0]['marketPlaceUrl']+'?supc='+item[0].get('identifier')

            else:

                pass 

            try:

                cashBack = getCashBack(item[0]['_id'], item[0]['source_id'], item[0]['category_id'])

                if not cashBack or cashBack.get('cash_back_status')!=1:

                    item[0]['cash_back_type'] = 0

                    item[0]['cash_back'] = 0

                else:

                    if cashBack.get('maxCashBack') is not None:

                        if cashBack.get('cash_back_type') ==1 and (float(cashBack.get('cash_back'))*item[0]['available_price'])/100 > cashBack.get('maxCashBack'):

                            cashBack['cash_back_type'] = 2

                            cashBack['cash_back'] = cashBack['maxCashBack']

                        elif cashBack.get('cash_back_type') ==2 and cashBack.get('cash_back') > cashBack.get('maxCashBack'):

                            cashBack['cash_back'] = cashBack['maxCashBack']

                    item[0]['cash_back_type'] = int(cashBack['cash_back_type'])

                    item[0]['cash_back'] = cashBack['cash_back']

            except:

                print "Error in adding cashback to deals"

                item[0]['cash_back_type'] = 0

                item[0]['cash_back'] = 0

            try:

                item[0]['dp'] = d['dp']

                item[0]['showDp'] = d['showDp']

            except:

                item[0]['dp'] = 0.0

                item[0]['showDp'] = 0

            try:

                item[0]['thumbnail'] = item[0]['thumbnail'].strip()

            except:

                pass

            try:

                if item[0]['showVideo']==0:

                    item[0]['videoLink'] =""

            except:

                item[0]['videoLink'] =""

            try:

                if item[0]['quantity'] >1:

                    ppq = float(item[0]['available_price'])/item[0]['quantity']

                    if ppq == round(ppq):

                        item[0]['ppq'] = int(ppq)

                    else:

                        item[0]['ppq'] = round(ppq,2)

                else:

                    item[0]['ppq'] = 0

            except:

                item[0]['ppq'] = 0

            if filterData is None:

                if item[0]['category_id']!=6:

                    try:

                        item[0]['filterLinkBrand'] = '/category/'+str(int(item[0]['category_id']))+'?searchFor='+urllib.quote(item[0]['brand'])+'&filter=brand&brands='+str(int(item[0]['brand_id']))

                        item[0]['filterTextBrand'] = (item[0]['brand'])

                    except:

                        pass

                else:

                    try:

                        if item[0]['subCategory']=='' or item[0]['subCategory'] is None or item[0]['subCategoryId']==0:

                            raise

                        item[0]['filterLinkSubCategory'] = '/category/'+str(int(item[0]['category_id']))+'?searchFor='+urllib.quote(item[0]['subCategory'])+'&filter=subcategory&subcategories='+str(int(item[0]['subCategoryId']))

                        item[0]['filterTextSubCategory'] = (item[0]['subCategory'])

                        item[0]['filterLinkBrand'] = '/category/'+str(int(item[0]['category_id']))+'?searchFor='+urllib.quote(item[0]['brand'])+'&filter=brand&brands='+str(int(item[0]['brand_id']))

                        item[0]['filterTextBrand'] = (item[0]['brand'])

                    except:

                        pass

            try:

                if item[0]['source_id']==4:

                    item_availability_info = mc.get("item_availability_"+str(item[0]['identifier']).strip())

                    if item_availability_info is None or len(item_availability_info)==0:

                        item[0]['availabilityInfo'] = [{}]

                    else:

                        item[0]['availabilityInfo'] = item_availability_info

            except:

                item[0]['availabilityInfo'] = [{}]

            if item[0]['source_id']==4:

                for av_info in item[0]['availabilityInfo']:

                    if not av_info:

                        continue

                    av_info['cash_back'] = item[0]['cash_back']

                    av_info['cash_back_type'] = item[0]['cash_back_type']

                    netPriceAfterCashBack = av_info['sellingPrice'] 

                    if av_info['cash_back_type'] ==1:

                        netPriceAfterCashBack = av_info['sellingPrice'] - math.floor(float(av_info['sellingPrice'])*av_info['cash_back']/100)

                    elif av_info['cash_back_type'] ==2:

                        netPriceAfterCashBack = av_info['sellingPrice'] - math.floor(av_info['cash_back'])

                    else:

                        pass

                    av_info['netPriceAfterCashBack'] = netPriceAfterCashBack

            #item[0]['netPriceAfterCashBack'] = d['netPriceAfterCashBack']

            if item[0]['source_id'] != SOURCE_MAP.get("PAYTM.COM"):

                if item[0]['codAvailable'] ==1:

                    paytmPrice = item[0]['available_price']

                else:

                    paytmPrice = item[0]['gross_price']

                if item[0]['cash_back_type'] ==1:

                    item[0]['netPriceAfterCashBack'] = paytmPrice -  math.floor(float(paytmPrice)*item[0]['cash_back']/100)

                elif item[0]['cash_back_type'] ==2:

                    item[0]['netPriceAfterCashBack'] = paytmPrice -  math.floor(item[0]['cash_back'])

                else:

                    item[0]['netPriceAfterCashBack'] = paytmPrice

            else:

                if item[0]['cash_back_type'] ==1:

                    item[0]['netPriceAfterCashBack'] = item[0]['available_price'] -  math.floor(float(item[0]['available_price'])*item[0]['cash_back']/100)

                elif item[0]['cash_back_type'] ==2:

                    item[0]['netPriceAfterCashBack'] = item[0]['available_price'] -  math.floor(item[0]['cash_back'])

                else:

                    item[0]['netPriceAfterCashBack'] = item[0]['available_price']

#             if specialOffer is not None and item[0]['source_id']==4:

#                 if item[0]['category_id'] in specialOffer.categories_applicable and item[0]['subCategoryId'] not in specialOffer.sub_categories_not_applicable:

#                     item[0]['specialOffer'] = specialOffer.__dict__

#             else:

#                 item[0]['specialOffer'] = {}

            temp.append(item[0])

        if len(temp) > 1:

            temp = sorted(temp, key = lambda x: (x['available_price']),reverse=False)

        dealsListMap.append(temp)

    return dealsListMap

def filterDeals(deals, filterData):

    print "Filter data ", filterData

    brandsFiltered = []

    subCategoryFiltered = []

    filterArray = filterData.split('|')

    for data in filterArray:

        try:

            filter, info = data.split(':')

        except Exception as ex:

            traceback.print_exc()

            continue

        if filter == 'brandFilter':

            toFilter = info.split('^')

            print "brand filter ",toFilter

            for deal in deals:

                if str(int(deal['brand_id'])) in toFilter:

                    brandsFiltered.append(deal)

        elif filter =='subCategoryFilter':

            toFilter = info.split('^')

            print "Sub Category filter ",toFilter

            for deal in deals:

                if str(int(deal['subCategoryId'])) in toFilter:

                    subCategoryFiltered.append(deal)