Subversion Repositories SmartDukaan

import MySQLdb
from dtr.storage.DataService import brand_preferences
from elixir import *
from dtr.storage import DataService, Mongo
from dtr.storage.DataService import user_filters
from sqlalchemy.sql.expression import func, between
from datetime import datetime,timedelta
from UserSegmentationLinksMaster import get_mongo_connection_new
import re
from dtr.utils.utils import to_java_date
DataService.initialize(echo=True)

DB_HOST = "localhost"
DB_USER = "root"
DB_PASSWORD = "shop2020"
DB_NAME = "dtr"

USER_SEGMENTATION_LINK_QUERY="""
select x.user_id,x.brand,x.category_id,x.count,x.source,x.weight 
  from
    (
    select c.user_id,c.brand,c.category_id,count(c.id) as count,"clicks" as source,(uw.weightage * count(c.id)) as weight 
    from clicks c join user_segmentation_weigthage uw on "clicks"=uw.weightage_type
    where  c.category_id is not null and c.category_id !=0 and c.brand is not null and 
    c.brand not like '' and c.user_id!=0 and datediff(curdate(),c.created)<7 group by c.category_id,c.brand,c.user_id
    
    UNION
   
    select m.userId as user_id,m.brand,m.categoryId as category_id,count(1) as count,"orders" as source,(uw.weightage * count(1)) as weight
    from merchantsuborders m join user_segmentation_weigthage uw on "orders"=uw.weightage_type 
    where m.categoryId is not null and m.brand is not null and m.brand not like 'NA' and m.userId!=0 and datediff(curdate(),m.createdTime)<7 group by m.categoryId,m.brand,m.userId
  
    UNION
  
    select fo.user_id,fo.brand,fo.category as category_id,count(fo.id) as count,"orders" as source, (uw.weightage * count(fo.id)) as weight
    from flipkartorders fo join user_segmentation_weigthage uw on "orders"=uw.weightage_type 
    where  fo.category is not null and fo.brand is not null and  fo.brand not like '' and fo.user_id!=0 and fo.user_id is not null 
    and datediff(curdate(),fo.created)<7 group by fo.category,fo.brand,fo.user_id
    
    UNION
    
    select bp.user_id,bp.brand,bp.category_id,count(1) as count,"preferences" as source,(uw.weightage* count(1)) as weight
    from brand_preferences bp join user_segmentation_weigthage uw on "preferences"=uw.weightage_type  
    where user_id!=0 and status='show' and datediff(curdate(),bp.created)<7 group by bp.category_id,bp.brand,bp.user_id
) as x

"""

USER_STORE_PRODUCT_VIEW="""
    SELECT u.user_id,count(u.id) AS count, SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(u.url,  '/', -1 ),'?',1),'%',1),'#',1)  
    AS skuBundleId,"product_view" as source,(uw.weightage* count(u.id)) as weight
    FROM user_urls u join user_segmentation_weigthage uw on "product_view"=uw.weightage_type 
    WHERE u.url LIKE '%store_products/view%' and u.user_id!=0 and 
    SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING_INDEX(u.url,  '/', -1 ),'?',1),'%',1),'#',1) not like ''
    and datediff(curdate(),u.created)<7
    group by u.user_id,skuBundleId; 
"""

REPLACE_QUERY="""
REPLACE INTO newuserlinkssegmentation( user_id, brand, source,category_id, count, weight,modified) 
SELECT user_id, brand, source,category_id, count, weight,created
FROM newuserlinkssegmentation7days
WHERE user_id =%s and brand=%s and category_id=%s and source=%s
"""


userMap = {} #key - user_id val - brand map
userMobileCategory={}
userTabletCategory={}

def getDbConnection():
    return MySQLdb.connect(DB_HOST, DB_USER, DB_PASSWORD, DB_NAME)


def deleteUserForLinks():
    print '********** Deleting records from last 7 days tables *********'
    conn = getDbConnection()
    cursor = conn.cursor()
    deleteAllUserGroups = "delete from newuserlinkssegmentation7days"
    try:
        cursor.execute(deleteAllUserGroups)
        conn.commit()
    except:
        conn.rollback()
    finally:
        conn.close()    
        print '********** Completed Deleting records from last 7 days tables *********'

def addUsersForLinks():
    print '********** Inserting records in last 7 days table *********'
    user_id=''
    brand=''
    category_id='' 
    source=''
    count=''
    weight=''

    datesql= USER_SEGMENTATION_LINK_QUERY
    conn = getDbConnection()
    cursor = conn.cursor()
    cursor.execute(datesql)
    result = cursor.fetchall()
    
    for r in result:
        user_id=r[0]
        brand=r[1]
        if r[2] == 'Mobiles':
            category_id=3
        elif r[2] == 'Tablets':
            category_id=5
        else:
            category_id=r[2] 
        count=r[3]
        source=r[4]
        weight=r[5]
        if brand is not 'NA':
            sql = "replace into newuserlinkssegmentation7days (user_id, brand, category_id, source, count,weight,created) values(%s, %s, %s, %s, %s,%s,%s)"
            cursor.execute(sql,(user_id,brand, category_id, source, count,weight,datetime.now()))
            conn.commit()
    conn.close()
    print '********** Completed inserting records for orders,preferences and clicks in last 7 days table *********'
    
def filters():
    print '********** Inserting records for filters in last 7 day table *********'
    conn = getDbConnection()
    cursor = conn.cursor()
    cursor.execute("select weightage from user_segmentation_weigthage where weightage_type='filters'")
    result = cursor.fetchall()
    filter_weight=result[0][0]
    todayDate=datetime.today().date()
    data = session.query((user_filters)).filter_by(type="brand").filter(~(user_filters.filters).like('NA')).filter(user_filters.created.between(todayDate-timedelta(days=6),todayDate)).all()
    for d in data:
        if userMap.has_key(d.user_id):
            brandMap = userMap.get(d.user_id) 
        else:
            brandMap = {}
        brands = d.filters.split('|')
        for brand in brands:
            if brandMap.has_key(brand):
                brandMap[brand] = brandMap.get(brand) + 1
            else:
                brandMap[brand] = 1
        userMap[d.user_id] = brandMap
    for x, y in userMap.iteritems():
        for bra,cow in y.iteritems():
            user_id=x
            brand=bra
            count=cow
            category_id=3
            weight=cow*filter_weight
            source='filters'
            if brand is not 'NA':
                sql = "replace into newuserlinkssegmentation7days (user_id, brand, category_id, source, count,weight,created) values(%s, %s, %s, %s, %s,%s,%s)"
                cursor.execute(sql,(user_id,brand, category_id, source, count,weight,datetime.now()))
                conn.commit()
    conn.close()
    print '********** Completed inserting records for filters in last 7 day table *********'
    
def storeProductView():
    print '********** Inserting records for store product view in last 7 day table *********'
    user_id=''
    brand=''
    category_id='' 
    source=''
    count=''
    weight=''
    
    result = get_mongo_connection_new().User.browsinghistories.find({"$and":[{"url":{"$regex" : "/store_products\/view/"}},{"created":{"$gte":getLastSeventhDay()}}]})
    
    for r in result:
        matchObj = re.search('store_products/view/(.+?)/(.+?)$',r.get('url'))
        if matchObj:
            skuBundleId = matchObj.group(2)
            if 'user_id' in skuBundleId or '=' in skuBundleId:
                newMatchObj = re.search('(.+?)\?',skuBundleId)
                if newMatchObj:
                    skuBundleId=newMatchObj.group(1)
                else:
                    print 'No match userId', skuBundleId
                    continue
            elif '#' in skuBundleId:
                newMatchObj = re.search('(.+?)\#',skuBundleId)
                if newMatchObj:
                    skuBundleId=newMatchObj.group(1)
                else:
                    print 'No match #', skuBundleId
                    continue
            if '%' in skuBundleId:
                newMatchObj = re.search('(.+?)\%',skuBundleId)
                if newMatchObj:
                    skuBundleId=newMatchObj.group(1)
                else:
                    print 'No match', skuBundleId                     
        else:
            print "No match!!"
        print skuBundleId            
        if skuBundleId.isdigit():
            print 'Inside is digit true'
            skuBundleId = int(skuBundleId)
        else:
            print 'Inside is digit false'
            skuBundleId =  int(float(skuBundleId))
            
        skuData = Mongo.getSkuBrandData(skuBundleId)
        if len(skuData) !=0:
            user_id=r.get('user_id')
            brand=skuData['brand']
            category_id =skuData['category_id']
            source='product_view'
            count=1
            if category_id==3:
                if userMobileCategory.has_key(user_id):
                    brandMap = userMobileCategory.get(user_id)
                else:
                    brandMap = {}
                if brandMap.has_key(brand):
                    brandMap[brand] = brandMap.get(brand) + 1 
                else:
                    brandMap[brand] = 1
                userMobileCategory[user_id] = brandMap
            if category_id==5:
                if userTabletCategory.has_key(user_id):
                    brandCategoryMap = userTabletCategory.get(user_id)
                else:
                    brandCategoryMap = {}
                if brandCategoryMap.has_key(brand):
                    brandCategoryMap[brand] = brandCategoryMap.get(brand) + 1
                else:
                    brandCategoryMap[brand] = 1
                userTabletCategory[user_id] = brandCategoryMap    
        else:
            print 'Sku bundle Id not present',r.get('url')

    conn=getDbConnection()
    cursor=conn.cursor()
    cursor.execute("select weightage from user_segmentation_weigthage where weightage_type='product_view'")
    result = cursor.fetchall()
    product_view_weight=result[0][0]
    
    for x, y in userMobileCategory.iteritems():
        for bra,cow in y.iteritems():
            user_id=x
            brand= bra
            count=cow
            weight=cow*product_view_weight
            source='product_view'
            category_id=3
            sql = "replace into newuserlinkssegmentation7days (user_id, brand, category_id, source, count,weight,created) values(%s, %s, %s, %s, %s,%s,%s)"
            cursor.execute(sql,(user_id,brand, category_id, source, count,weight,datetime.now()))
            conn.commit()
            
    for x, y in userTabletCategory.iteritems():
        for bra,cow in y.iteritems():
            user_id=x
            brand= bra
            count=cow
            weight=cow*product_view_weight
            source='product_view'
            category_id=5
            sql = "replace into newuserlinkssegmentation7days (user_id, brand, category_id, source, count,weight,created) values(%s, %s, %s, %s, %s,%s,%s)"
            cursor.execute(sql,(user_id,brand, category_id, source, count,weight,datetime.now()))
            conn.commit()
    conn.close()
    
    print '********** Completed inserting records for store product view in last 7 day table *********'
    
def findRepeatingOrders():
    print '********** Call to sanitize data for orders if any multiple *********'
    datesql= "select user_id,brand,category_id from userlinkssegmentation7days where source='orders' group by user_id,brand,category_id having count(source)=2;"
    conn = getDbConnection()
    cursor = conn.cursor()
    rows_count = cursor.execute(datesql)
    if rows_count>0:
        result = cursor.fetchall()
        for r in result:
            print r
            findDetails(r[0],r[1],r[2])
    else:
        print 'No Data to be sanitized for repeating orders'            
        
def findDetails(userId,brand,category):
    datesql= "select sum(count),sum(weight) from userlinkssegmentation7days where user_id=%s and source='orders' and brand=%s group by source;"
    conn = getDbConnection()
    cursor = conn.cursor()
    cursor.execute(datesql,(userId,brand))
    result = cursor.fetchall()
    for r in result:
        deletesql="delete from userlinkssegmentation7days where user_id=%s and brand=%s and source='orders'"
        cursor = conn.cursor()
        cursor.execute(deletesql,(userId,brand))
        conn.commit()
        insertsql="insert into userlinkssegmentation7days (user_id, brand, category_id, source, count,weight) values(%s, %s, %s, %s, %s,%s)"
        cursor = conn.cursor()
        cursor.execute(insertsql,(userId,brand,category,'orders',r[0],r[1]))
        conn.commit()

def findRepeatingClicks():
    print '********** Call to sanitize data for product view if any multiple *********'
    datesql= "select user_id,brand,category_id from userlinkssegmentation7days where source='product_view' group by user_id,brand,category_id having count(source)=2;"
    conn = getDbConnection()
    cursor = conn.cursor()
    rows_count = cursor.execute(datesql)
    if rows_count>0:
        result = cursor.fetchall()
        for r in result:
            print r
            findViewDetails(r[0],r[1],r[2])
    else:
        print 'No Data to be sanitized for product view'            
        
def findViewDetails(userId,brand,category):
    datesql= "select sum(count),sum(weight) from userlinkssegmentation7days where user_id=%s and source='product_view' and brand=%s group by source;"
    conn = getDbConnection()
    cursor = conn.cursor()
    cursor.execute(datesql,(userId,brand))
    result = cursor.fetchall()
    for r in result:
        deletesql="delete from userlinkssegmentation7days where user_id=%s and brand=%s and source='product_view'"
        cursor = conn.cursor()
        cursor.execute(deletesql,(userId,brand))
        conn.commit()
        insertsql="insert into userlinkssegmentation7days (user_id, brand, category_id, source, count,weight) values(%s, %s, %s, %s, %s,%s)"
        cursor = conn.cursor()
        cursor.execute(insertsql,(userId,brand,category,'product_view',r[0],r[1]))
        conn.commit()        
    
    
def checkCountForUpdate():
    print '********** Checking users brand whose cumulative weight is greater than 3 *********'
    datesql= "select user_id,brand,category_id from newuserlinkssegmentation7days group by user_id,brand,category_id having sum(weight) >=3;"
    conn = getDbConnection()
    cursor = conn.cursor()
    cursor.execute(datesql)
    result = cursor.fetchall()
    for r in result:
        checkUserForUpdate(r[0],r[1],r[2])
    conn.close()    
def checkUserForUpdate(userId,brand,category):
    print '********** Checking users brand present or not in the master table *********'
    sources = ['product_view','clicks','filters','orders','preferences']
    datesql= "select distinct brand from newuserlinkssegmentation where user_id=%s and brand=%s and category_id=%s" 
    conn = getDbConnection()
    cursor = conn.cursor()
    cursor.execute(datesql,(userId,brand,category))
    result = cursor.fetchone()
    print 'Result in Check user for update',result
    if len(result)==0:
        insertNewUserLinkDetails(userId,brand,category)
    else:
        datesql= "select distinct source from newuserlinkssegmentation where user_id=%s and brand=%s and category_id=%s"
        cursor.execute(datesql,(userId,result[0][0],category))
        result = cursor.fetchall()
        for source in result:
            userReplaceData(userId,brand,category, source)
            sources.remove(source)
        for source in sources:
            insertNewUserLinkDetailsSourceWise(userId,brand,category, source)
    conn.close()    

def userReplaceData(userId,brand,category,source):
    print 'Data that needs to be updated for', userId,brand,category
    conn = getDbConnection()
    cursor = conn.cursor()
    print REPLACE_QUERY%(userId,brand,category,source)
    cursor.execute(REPLACE_QUERY,(userId,brand,category,source))
    conn.commit()
    conn.close()

def insertNewUserLinkDetails(userId,brand,category):
    print 'Data needs to be inserted as the brand was not present earlier', userId,brand,category
    conn = getDbConnection()
    selectNewUser="select * from newuserlinkssegmentation7days where user_id=%s and brand=%s and category_id=%s"
    cursor = conn.cursor()
    cursor.execute(selectNewUser,(userId,brand,category))
    result = cursor.fetchall()
    print 'Result while inserting new record in master',result
    for r in result:
        sql = "insert into newuserlinkssegmentation (user_id, brand, category_id, source, count,weight,created,modified) values(%s, %s, %s, %s, %s,%s,%s,%s)"
        cursor.execute(sql,(r[0],r[1],r[2], r[3], r[4],r[5],datetime.now(),datetime.now()))
        conn.commit()
    conn.close()
    
def insertNewUserLinkDetailsSourceWise(userId,brand,category,source):
    print 'Data needs to be inserted as the source was not present earlier', userId,brand,category,source
    conn = getDbConnection()
    selectNewUser="select * from newuserlinkssegmentation7days where user_id=%s and brand=%s and category_id=%s and source=%s"
    cursor = conn.cursor()
    cursor.execute(selectNewUser,(userId,brand,category,source))
    result = cursor.fetchall()
    print 'Result while inserting new record in master',result
    for r in result:
        sql = "insert into newuserlinkssegmentation (user_id, brand, category_id, source, count,weight,created,modified) values(%s, %s, %s, %s, %s,%s,%s,%s)"
        cursor.execute(sql,(r[0],r[1],r[2], r[3], r[4],r[5],datetime.now(),datetime.now()))
        conn.commit()
    conn.close()    
    
def getLastSeventhDay():
    return to_java_date(datetime.today() - timedelta(days=7))
            
def main():
    print datetime.now()
    print '********** Starting to update links for Users *********'
    deleteUserForLinks()
    addUsersForLinks()
    filters()
    storeProductView()
    #findRepeatingOrders()
    #findRepeatingClicks()
    checkCountForUpdate()
    print '********** Finished updating records for user links *********'
if __name__=='__main__':
    main()