Subversion Repositories SmartDukaan

#!/usr/bin/python

import threading

import time

import MySQLdb

from elixir import *

from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound

from sqlalchemy.sql import func

from sqlalchemy.sql.expression import and_, or_, desc, not_, distinct, cast, \

    between

from urlparse import urlparse

from urlparse import parse_qs

import requests

import json

import optparse

import urllib2

import base64

import urllib

import logging

from dtr.utils.utils import get_mongo_connection, to_java_date

from datetime import datetime

GCM_URL = "https://android.googleapis.com/gcm/send"

GOOGLE_API_KEY = "AIzaSyDw1qBnmxtnfR9NqBewryQ-yo3cG2ravGM"

headers = {'content-type':'application/json', "authorization":"key=" + GOOGLE_API_KEY}

aff_url_headers = { 

            'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',

            'Connection':'keep-alive'

        }

mongoHost = 'localhost' 

campaignUsersMap = {}

notificationCampaignsMap = {}

pendingNotificationEntryMap = {}

userGcmRegIdMap = {}

db = MySQLdb.connect('localhost',"root","shop2020","dtr" )

cursor = db.cursor()

ALL_STORES_SQL = "select * from stores"

GCM_REG_ID_SQL1 = "select x.user_id, x.gcm_regid, x.id, x.androidid, x.created  from (select * from gcm_users where user_id in (%s) and androidid is not null order by id desc) as x group by x.user_id, x.gcm_regid, x.androidid"

GCM_REG_ID_SQL2 = "select x.user_id, x.gcm_regid, x.id, x.androidid, x.created, x.imeinumber  from (select * from gcm_users where user_id in (%s) and androidid is null and imeinumber is not null order by id desc) as x group by x.user_id, x.imeinumber"

cursor.execute(ALL_STORES_SQL)

result_stores = cursor.fetchall()

domainStoresMap = {}

for rec in result_stores:

    domainStoresMap[rec[2]] = rec

logging.basicConfig(level=logging.DEBUG,

                    format='[%(levelname)s] (%(threadName)-10s) %(message)s',

                    )

class NotificationThread (threading.Thread):

    def __init__(self, threadID, name, recordsList):

        threading.Thread.__init__(self)

        self.threadID = threadID

        self.name = name

        self.recordsList = recordsList

    def run(self):

        logging.debug('Starting')

        handleCampaignRequest(self.name, self.recordsList)

        logging.debug('Completed')

def handleCampaignRequest(threadName, recordsList ):

    for record in recordsList:

        userGcmRegIdDetails = userGcmRegIdMap.get(long(record.get('user_id')))

        campaign = notificationCampaignsMap.get(long(record.get('notification_campaign_id'))) 

        gcm_id = record.get('gcm_id')

        detailsMap = userGcmRegIdDetails.get(gcm_id)

        result_url = ""

        if campaign.get('type')=='url':

            parsed_uri = urlparse(campaign.get('url'))

            domain = '{uri.netloc}'.format(uri=parsed_uri)

            logging.debug('Affiliate Domain:-'+str(domain))

            logging.debug('User Id:-'+str(record.get('user_id'))+' And GCM Reg Id:- '+ str(detailsMap.get('gcm_regid')))

            store = domainStoresMap.get(domain)

            if store is not None:

                url_params = { 'url' : campaign.get('url'),  'userId' : record.get('user_id'), 'storeId' : store[0] }

                encoded_url_params = urllib.urlencode(url_params)

                DTR_API_BASIC_AUTH = base64.encodestring('%s:%s' % ("dtr", "dtr18Feb2015")).replace('\n', '')

                pushpostrequest = urllib2.Request('http://api.profittill.com/pushnotifications/generateAffiliateUrl', encoded_url_params, headers=aff_url_headers)

                pushpostrequest.add_header("Authorization", "Basic %s" % DTR_API_BASIC_AUTH)

                json_result =  json.loads(urllib2.urlopen(pushpostrequest).read())

                result_url = json_result['url']

                logging.debug('User Id:-'+str(record.get('user_id'))+' Notification Url:- '+ str(result_url))

            else:

                queryString = urlparse(campaign.get('url').strip()).query

                parsed_url = parse_qs(queryString)

                if not parsed_url.has_key('user_id'):

                    if len(queryString)>0:

                        result_url = campaign.get('url').strip()+'&user_id='+str(record.get('user_id'))

                        logging.debug('User Id:-'+str(record.get('user_id'))+' Notification Url:- '+ str(result_url))

                    else:

                        result_url = campaign.get('url').strip()+'?user_id='+str(record.get('user_id'))

                        logging.debug('User Id:-'+str(record.get('user_id'))+' Notification Url:- '+ str(result_url))

                else:

                    logging.debug('User Id:-'+str(record.get('user_id'))+' Notification Url:- '+ str(record.get('user_id')))

        if campaign.get('url') is None or str(campaign.get('url'))=='':

            result_url = 'http://api.profittill.com/deals?user_id='+str(record.get('user_id'))

        data = {"message":campaign.get('message'),"cid":str(campaign.get('_id'))+"_"+str(record.get('_id')),"title":campaign.get('title'),

                "type":campaign.get('type'),"url":result_url.strip(),"vibrate":1,"sound":1,"largeIcon":"large_icon",

                "smallIcon":"small_icon","priority":"high","time_to_live":long(campaign.get('expiresat'))-long(time.mktime(datetime.now().timetuple()))}

        post_data = {}

        post_data['data'] = data

        regIds = []

        regIds.append(detailsMap.get('gcm_regid'))

        post_data['registration_ids'] = regIds

        post_data_json = json.dumps(post_data)

        logging.debug('User Id:- '+str(record.get('user_id'))+' Post Data Json :- '+str(post_data_json))

        response = requests.post(GCM_URL, data=post_data_json, headers=headers)

        logging.debug('User Id:- '+str(record.get('user_id'))+' GCM_ID:- '+str(gcm_id)+' Response :-'+str(response.text))

        result = json.loads(response.text)

        if result["success"]:

            get_mongo_connection(host=mongoHost).User.pushnotifications.update({'_id':record.get('_id')},{"$set":{'message':'success','type':'sent','sent_timestamp':to_java_date(datetime.now())}})

            logging.debug('User Id:- '+str(record.get('user_id'))+' GCM_ID:- '+str(gcm_id)+' Update Response :- Notification Success True')

        else:

            get_mongo_connection(host=mongoHost).User.pushnotifications.update({'_id':record.get('_id')},{"$set":{'message':result["results"][0]["error"],'type':'failed','sent_timestamp':to_java_date(datetime.now())}})

            logging.debug('User Id:- '+str(record.get('user_id'))+' GCM_ID:- '+str(gcm_id)+' Update Response :- Notification Fail True')

            updateGcmUserSql = "update gcm_users set failurecount=failurecount+1 where gcm_regid='%s'"%(detailsMap.get('gcm_regid'))

            logging.debug('Update GCM User Query :-'+str(updateGcmUserSql))

            try:

                dtrdb = MySQLdb.connect('localhost',"root","shop2020","dtr" )

                cursor = dtrdb.cursor()

                cursor.execute(updateGcmUserSql)

                dtrdb.commit()

                session.commit()

                dtrdb.close()

            except:

                dtrdb.rollback()

                dtrdb.close()

def chunks(l, n):

    """Yield successive n-sized chunks from l."""

    for i in xrange(0, len(l), n):

        yield l[i:i+n]

class __PushNotification:

    def __init__(self, notification_campaign_id, user_id, message, type, sms_type, sms_id, android_id, pushed_by, sent_timestamp, receive_timestamp, open_timestamp, sms_timestamp, gcm_id, created_timestamp, gcm_expired, notification_accounted):

        self.notification_campaign_id = notification_campaign_id

        self.user_id = user_id

        self.message = message

        self.type = type

        self.sms_type = sms_type

        self.sms_id = sms_id

        self.android_id = android_id

        self.pushed_by = pushed_by

        self.sent_timestamp = sent_timestamp

        self.receive_timestamp = receive_timestamp

        self.open_timestamp = open_timestamp

        self.sms_timestamp = sms_timestamp

        self.gcm_id = gcm_id

        self.created_timestamp = created_timestamp

        self.gcm_expired = gcm_expired

        self.notification_accounted = notification_accounted

def getPendingCampaigns():

    campaigns = list(get_mongo_connection(host=mongoHost).User.notificationcampaigns.find({'notification_processed':0,'expiresat':{'$gte':to_java_date(datetime.now())}}))

    return campaigns

def populateCampaignsMap(pendingCampaigns):

    global campaignUsersMap

    global notificationCampaignsMap

    for campaign in pendingCampaigns:

        cursor.execute(str(campaign['sql']))

        user_records = cursor.fetchall()

        userids = []

        for record in user_records:

            if str(record[0]) not in userids:

                userids.append(str(record[0]))

        campaignUsersMap[long(campaign['_id'])]=userids

        notificationCampaignsMap[long(campaign['_id'])] = campaign

def insertPushNotificationEntriesToSent():

    global userGcmRegIdMap

    for campaignId, userList in campaignUsersMap.items():

        logging.debug("CampaignId:- "+str(campaignId))

        if len(userList)==0:

            continue

        logging.debug("GCM_REG_SQL_1:- "+GCM_REG_ID_SQL1%(",".join(map(str,userList))))

        cursor.execute(GCM_REG_ID_SQL1%(",".join(map(str,userList))))

        result_data = cursor.fetchall()

        if result_data and len(result_data)>0:

            user_list = []

            for userId in userList:

                user_list.append(userId)

            for dataRec in result_data:

                '''

                if str(dataRec[0]) in user_list:

                    user_list.remove(str(dataRec[0]))

                '''

                if userGcmRegIdMap.has_key(dataRec[0]):

                    detailMap = {}

                    gcmRegIdMap = userGcmRegIdMap.get(dataRec[0])

                    detailMap['gcm_regid'] = dataRec[1]

                    if dataRec[3] is not None:

                        detailMap['android_id'] = dataRec[3]

                    gcmRegIdMap[dataRec[2]]= detailMap

                    userGcmRegIdMap[dataRec[0]] = gcmRegIdMap

                else:

                    gcmRegIdMap = {}

                    detailMap = {}

                    detailMap['gcm_regid'] = dataRec[1]

                    if dataRec[3] is not None:

                        detailMap['android_id'] = dataRec[3]

                    gcmRegIdMap[dataRec[2]]= detailMap

                    userGcmRegIdMap[dataRec[0]] = gcmRegIdMap

            logging.debug("Old Users.."+str(user_list))

            logging.debug("GCM_REG_SQL_2:- "+GCM_REG_ID_SQL2%(",".join(map(str,user_list))))       

            cursor.execute(GCM_REG_ID_SQL2%(",".join(map(str,user_list))))

            result_data = cursor.fetchall()

            for dataRec in result_data:

                if userGcmRegIdMap.has_key(dataRec[0]):

                    detailMap = {}

                    gcmRegIdMap = userGcmRegIdMap.get(dataRec[0])

                    detailMap['gcm_regid'] = dataRec[1]

                    if dataRec[3] is not None:

                        detailMap['android_id'] = dataRec[3]

                    gcmRegIdMap[dataRec[2]]= detailMap

                    userGcmRegIdMap[dataRec[0]] = gcmRegIdMap

                else:

                    gcmRegIdMap = {}

                    detailMap = {}

                    detailMap['gcm_regid'] = dataRec[1]

                    if dataRec[3] is not None:

                        detailMap['android_id'] = dataRec[3]

                    gcmRegIdMap[dataRec[2]]= detailMap

                    userGcmRegIdMap[dataRec[0]] = gcmRegIdMap

        else:

            logging.debug("GCM_REG_SQL_2:- "+GCM_REG_ID_SQL2%(",".join(map(str,userList))))

            cursor.execute(GCM_REG_ID_SQL2%(",".join(map(str,userList))))   

            result_data = cursor.fetchall()

            for dataRec in result_data:

                if userGcmRegIdMap.has_key(dataRec[0]):

                    detailMap = {}

                    gcmRegIdMap = userGcmRegIdMap.get(dataRec[0])

                    detailMap['gcm_regid'] = dataRec[1]

                    if dataRec[3] is not None:

                        detailMap['android_id'] = dataRec[3]

                    gcmRegIdMap[dataRec[2]]= detailMap

                    userGcmRegIdMap[dataRec[0]] = gcmRegIdMap

                else:

                    gcmRegIdMap = {}

                    detailMap = {}

                    detailMap['gcm_regid'] = dataRec[1]

                    if dataRec[3] is not None:

                        detailMap['android_id'] = dataRec[3]

                    gcmRegIdMap[dataRec[2]]= detailMap

                    userGcmRegIdMap[dataRec[0]] = gcmRegIdMap   

    logging.debug("CampaignUsersMap"+str(campaignUsersMap)) 

    for campaignId, userList in campaignUsersMap.items():

        for userId in userList:

            gcmRegIdMap = userGcmRegIdMap.get(long(userId))

            if gcmRegIdMap is None:

                gcmRegIdMap = {}

            for gcm_id, details in gcmRegIdMap.items():

                android_id = None

                logging.debug("User Id:- "+str(userId)+" ..User Details:- "+str(details))

                if details.has_key('android_id'):

                    android_id = details['android_id']

                pushNotificationObj = __PushNotification(long(campaignId), long(userId), None, 'pending', \

                                    None, None, android_id, "php", None, None, None, None, gcm_id, to_java_date(datetime.now()),0,0)

                get_mongo_connection(host=mongoHost).User.pushnotifications.insert(pushNotificationObj.__dict__)

def populatePendingNotificationEntriesToBeSent():

    global pendingNotificationEntryMap

    pendingNotificationEntries = list(get_mongo_connection(host=mongoHost).User.pushnotifications.find({'type':'pending'}))

    for entry in pendingNotificationEntries:

        if pendingNotificationEntryMap.has_key(entry['notification_campaign_id']):

            entries = pendingNotificationEntryMap.get(entry['notification_campaign_id'])

            entries.append(entry)

            pendingNotificationEntryMap[entry['notification_campaign_id']] = entries

        else:

            entries = []

            entries.append(entry)

            pendingNotificationEntryMap[entry['notification_campaign_id']] = entries

def initiateNotificationThreadProcess(chunkSize):

    count = 1

    logging.debug('Starting Push Notification Job....'+str(datetime.now()))

    for entries in pendingNotificationEntryMap.values():

        campaign_receivers_list = list(chunks(entries, chunkSize))

        print len(campaign_receivers_list)

        for sublist in campaign_receivers_list:

            thread = NotificationThread(count, "Thread-"+str(count), sublist)

            thread.start()

            count = count +1

    logging.debug('Stopping Push Notification Job....'+str(datetime.now()))

def markNotificationCampaignsProcessed():

    for campaign in notificationCampaignsMap.values():

        logging.debug('Notification Campaign....'+str(campaign.get('_id'))+"...Marked Processed. "+str(datetime.now()))

        get_mongo_connection(host=mongoHost).User.notificationcampaigns.update({'_id':campaign.get('_id')},{"$set":{'notification_processed':1}})     

def main():

    global mongoHost

    parser = optparse.OptionParser()

    parser.add_option("-C", "--chunksize", dest="chunksize",

                      default="100",

                      type="int", help="The requsets a single thread handles",

                      metavar="CHUNKSIZE")

    parser.add_option("-M", "--mongo_host", dest="mongo_host",

                      default="localhost",

                      type="str", help="The requsets a single thread handles",

                      metavar="MONGOHOST")

    (options, args) = parser.parse_args()

    mongoHost = options.mongo_host

    pendingCampaigns = getPendingCampaigns()

    populateCampaignsMap(pendingCampaigns)

    insertPushNotificationEntriesToSent()

    populatePendingNotificationEntriesToBeSent()

    initiateNotificationThreadProcess(options.chunksize)

    markNotificationCampaignsProcessed()

    db.close()

if __name__=='__main__':

    main()