Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from datetime import datetime, timedelta

from dtr.config import PythonPropertyReader

from dtr.storage.Mongo import getDealRank

from dtr.storage.Mysql import getOrdersAfterDate1, getOrdersAfterDate

from pprint import pprint

from pymongo.mongo_client import MongoClient

import base64

import importlib

import json

import math

import mechanize

import time

import traceback

import urllib

import urllib2

sourceMap = {1:"amazon", 2:"flipkart", 3:"snapdeal", 4:"spice", 5:"shopclues", 6:"paytm"}

headers = { 

           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'

        }

CASHBACK_URL = PythonPropertyReader.getConfig('CASHBACK_URL')

USER_LOOKUP_URL = PythonPropertyReader.getConfig('USER_LOOKUP_URL')

WALLET_CREDIT_URL = PythonPropertyReader.getConfig('WALLET_CREDIT_URL')

def getStore(source_id):

    #module = sourceMap[source_id]

    store = Store(source_id)

    try:

        module = importlib.import_module("dtr.sources." + sourceMap[source_id])

        store = getattr(module, "Store")(source_id)

        return store

    except:

        traceback.print_exc()

        return None

class ScrapeException(Exception):

    """Exception raised for errors in the input.

    Attributes:

        expr -- input expression in which the error occurred

        msg  -- explanation of the error

    """

    def __init__(self, expr, msg):

        self.expr = expr

        self.msg = msg

class ParseException(Exception):

    """Exception raised for errors in the input.

    Attributes:

        expr -- input expression in which the error occurred

        msg  -- explanation of the error

    """

    def __init__(self, expr, msg):

        self.expr = expr

        self.msg = msg

client = MongoClient('mongodb://localhost:27017/') 

class Store(object):

    ORDER_PLACED = 'Order Placed'

    ORDER_DELIVERED = 'Delivered'

    ORDER_SHIPPED = 'Shipped' #Lets see if we can make use of it

    ORDER_CANCELLED = 'Cancelled'

    CB_INIT = 'Waiting Confirmation'

    CB_PENDING = 'Pending'

    CB_CREDIT_IN_PROCESS = 'Credit in process'

    CB_CREDITED = 'Credited to wallet'

    CB_NA = 'Not Applicable'

    CB_APPROVED = 'Approved'

    CB_REJECTED = 'Rejected'

    CB_ONHOLD = 'On hold'

    CB_CANCELLED = 'Cancelled'

    CONF_CB_SELLING_PRICE = 0

    CONF_CB_DISCOUNTED_PRICE = 1

    def __init__(self, store_id):

        self.db = client.Dtr

        self.store_id = store_id

        self.store_name = sourceMap[store_id]

    '''

    To Settle payback for respective stores.

    Also ensures that settlement happens only for approved orders

    '''

    def getName(self):

        raise NotImplementedError

    def scrapeAffiliate(self, startDate=None, endDate=None):

        raise NotImplementedError

    def getTrackingUrls(self, userId):

        raise NotImplementedError

    def saveToAffiliate(self, offers):

        raise NotImplementedError

    def getUpdateMap(self, subOrder, cashBackStatus):

        updateMap = {}

        closed = False

        for (key,value) in subOrder.iteritems():

            if key=="status":

                if value==Store.ORDER_CANCELLED:

                    closed = True

                    updateMap['subOrders.$.closed'] = True

                    if cashBackStatus == Store.CB_PENDING:

                        updateMap['subOrders.$.cashBackStatus'] = Store.CB_CANCELLED

                if value==Store.ORDER_DELIVERED:

                    closed = True

                    updateMap['subOrders.$.closed'] = True

                    if cashBackStatus == Store.CB_PENDING:

                        updateMap['subOrders.$.cashBackStatus'] = Store.CB_APPROVED

            updateMap['subOrders.$.' + key] = value

        subOrder['closed'] = closed

        return updateMap

    def saveOrder(self, merchantOrder):

        #merchantOrder = Order()

        #merchantOrder.status

        #mercha

        pass

    def scrapeStoreOrders(self,):

        raise NotImplementedError

    def updateSubOrder(self, subOrder):

        pass

        #if subOrder.get

    def _saveToOrder(self, order):

        collection = self.db.merchantOrder

        try:

            order = collection.insert(order)

            return True 

        except Exception as e:

            return False

    def _updateToOrder(self, order):

        collection = self.db.merchantOrder

        try:

            print "************************************************************************"

            print "order"

            print order

            print "************************************************************************"

            collection.update({"orderId":order['orderId']},{"$set":order}, upsert = True)

            #merchantOder 

        except Exception as e:

            traceback.print_exc()

    def getCashbackAmount(self, productCode, amount):

        alagvar = CASHBACK_URL % (productCode,self.store_id)

        filehandle = urllib2.Request(alagvar,headers=headers)

        x= urllib2.urlopen(filehandle)

        rmap = json.loads(str(x.read()))

        if len(rmap)==0:

            return (0,0)

        else:

            if rmap['cash_back_description'] == 'PERCENTAGE':

                return (math.floor((amount * rmap['cash_back'])/100), rmap['cash_back'])

            else:

                return (rmap['cash_back'], 0)

    '''

    Parses the order for specific store

    order id, total amount, created on(now() if could not parse

    suborder id, title, quantity, unit price, expected delivery date,

    status (default would be Order placed)

    once products are identified, each suborder can then be updated

    with respective cashback.

    Possible fields to display for Not yet delivered orders are 

    Product/Quantity/Amount/Store/CashbackAmount/OrderDate/ExpectedDelivery/OrderStaus/DetailedStatus/CashbackStatus

    No need to show cancelled orders.

    CashbackStatus - NotApplicable/Pending/Approved/Cancelled/CreditedToWallet

    OrderStatus - Placed/Cancelled/Delivered

    '''

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        pass

    def _updateOrdersPayBackStatus(self, searchMap, updateMap):

        searchMap['subOrders.missingAff'] = False

        updateMap['subOrders.$.missingAff'] = True

        self.db.merchantOrder.update(searchMap, { '$set': updateMap }, multi=True)

    def _getActiveOrders(self, searchMap={}, collectionMap={}):

        collection = self.db.merchantOrder

        searchMap = dict(searchMap.items()+ {"closed": False, "storeId" : self.store_id}.items()) 

        collectionMap =  dict(collectionMap.items() + {"orderSuccessUrl":1, "orderId":1,"subOrders":1, "placedOn":1}.items())

        stores = collection.find(searchMap, collectionMap)

        return [store for store in stores]

    def _getMissingOrders(self,searchMap={}):

        collection = self.db.merchantOrder

        searchMap = dict(searchMap.items()+ {"requireDetail":True, "storeId":self.store_id}.items()) 

        orders = collection.find(searchMap)

        return list(orders)

    def _isSubOrderActive(self,order, merchantSubOrderId):

        subOrders = order.get("subOrders")

        if subOrders is None:

            return None

        for subOrder in subOrders:

            if merchantSubOrderId == subOrder.get("merchantSubOrderId"):

                return subOrder

        return None

    def populateDerivedFields(self, order):

        closed=True

        for subOrder in order.subOrders:

            if subOrder.closed:

                continue

            cashbackAmount, cashbackPercent = self.getCashbackAmount(subOrder.productCode, subOrder.amountPaid)

            cashbackStatus = Store.CB_PENDING

            if cashbackAmount <= 0:

                cashbackStatus = Store.CB_NA

            subOrder.cashBackAmount = cashbackAmount

            subOrder.cashBackPercentage = cashbackPercent

            subOrder.cashBackStatus = cashbackStatus      

            subOrder.closed = subOrder.status in [self.ORDER_CANCELLED, self.ORDER_DELIVERED]

            closed = closed and subOrder.closed

            dealRank = getDealRank(subOrder.productCode, self.store_id, order.userId)

            subOrder.dealRank = dealRank.get('rank')

            subOrder.rankDesc = dealRank.get('description')

            subOrder.maxNlc = dealRank.get('maxNlc')

            subOrder.minNlc = dealRank.get('minNlc')

            subOrder.db = dealRank.get('dp')

            subOrder.itemStatus = dealRank.get('status')

        order.closed = closed

def settlePayBack(runtype='dry'):

    userAmountMap = {}

    searchMapList = []

    for mo in client.Dtr.merchantOrder.find({"subOrders.cashBackStatus":Store.CB_APPROVED}):

        userId = mo.get("userId")

        if   mo.get('subOrders') is not None:

            for so in mo['subOrders']:

                if so.get('cashBackStatus') == Store.CB_APPROVED:

                    searchMapList.append({"orderId":mo.get("orderId"), "subOrders.merchantSubOrderId":so.get("merchantSubOrderId")})

                    if not userAmountMap.has_key(userId):

                        userAmountMap[userId] = so.get('cashBackAmount')

                    else:

                        userAmountMap[userId] += so.get('cashBackAmount')

                    print "%s\t%s\t%s\t%s\t%s\t%s\t%s"%(userId, mo.get("orderId"), so.get("merchantSubOrderId"),so.get("productTitle") ,so.get("cashBackStatus"), so.get("cashBackAmount"), so.get("batchId"))

                else:  

                    print "%s\t%s\t%s\t%s\t%s\t%s\t%s"%(userId, mo.get("orderId"), so.get("merchantSubOrderId"),so.get("productTitle") ,so.get("cashBackStatus"), so.get("cashBackAmount"), so.get("batchId"))

    for searchMap in searchMapList:

        print "%s\t%s"%(searchMap.get('orderId'),searchMap.get('subOrders.merchantSubOrderId'))

    for key,val in userAmountMap.iteritems():

        print "%s\t%s"%(key,val)

    if (runtype=='live'):

        bulk = client.Dtr.merchantOrder.initialize_ordered_bulk_op()

        if len(searchMapList) == 0:

            return

        for searchMap in searchMapList:

            bulk.find(searchMap).update({'$set' : {'subOrders.$.cashBackStatus':Store.CB_CREDIT_IN_PROCESS}})

        bulk.execute()

        datetimeNow = datetime.now() 

        batchId = int(time.mktime(datetimeNow.timetuple()))

        if refundToWallet(batchId, userAmountMap):

            bulk = client.Dtr.merchantOrder.initialize_ordered_bulk_op()

            for searchMap in searchMapList:

                bulk.find(searchMap).update({'$set' : {'subOrders.$.cashBackStatus':Store.CB_CREDITED, "subOrders.$.batchId":batchId}})

            print bulk.execute()

            for key, value in userAmountMap.iteritems():

                client.Dtr.refund.insert({"userId": key, "batch":batchId, "userAmount":value, "timestamp":datetime.strftime(datetimeNow,"%Y-%m-%d %H:%M:%S")})

                client.Dtr.user.update({"userId":key}, {'$inc': { "credited": value}}, upsert=True)

            tprint("PayBack Settled")

        else:

            tprint("Error Occurred while running batch. Rolling Back")

            bulk = client.Dtr.merchantOrder.initialize_ordered_bulk_op()

            for searchMap in searchMapList:

                bulk.find(searchMap).update({'$set' : {'subOrders.$.cashBackStatus':Store.CB_APPROVED}})

            bulk.execute()

def settlePayBack1(runtype='dry'):

        approvedUserMap = {}

        pendingUserMap = {}

        creditedToWalletUserMap = {}

        #client.Dtr.merchantOrder.update({'subOrders.cashBackStatus':Store.CB_APPROVED},{'$set':{'subOrders.$.cashBackStatus':Store.CB_CREDIT_IN_PROCESS}}, multi=True)    

        for mo in client.Dtr.merchantOrder.find({"subOrders.cashBackStatus":Store.CB_CREDITED}):

            userId = mo.get("userId")

            for so in mo['subOrders']:

                print "%s\t%s\t%s\t%s\t%s\t%s\t%s"%(userId, mo.get("orderId"), so.get("merchantSubOrderId"),so.get("productTitle") ,so.get("cashBackStatus"), so.get("cashBackAmount"), so.get("batchId"))

        for refund in client.Dtr.refund.find():

            print "%s\t%s\t%s\t%s"%(refund.get("userId") ,refund.get("timestamp") ,refund.get("userAmount"), refund.get("batch"))

def refundToWallet(batchId, userAmountMap):

    base64string = base64.encodestring('%s:%s' % ("dtr", "dtr18Feb2015")).replace('\n', '')

    batchUpdateMap = {}

    try :

        saholicUserAmountMap = {}

        for key, value in userAmountMap.iteritems():

            userLookupRequest = urllib2.Request(USER_LOOKUP_URL %(key), headers=headers)

            userLookupRequest.add_header("Authorization", "Basic %s" % base64string)

            try:

                response = urllib2.urlopen(userLookupRequest).read()

                saholicUserId = json.loads(response)['account_key']

                saholicUserAmountMap[saholicUserId] = value

            except:

                tprint("Could not fetch saholic id for user : " + str(key))

                continue

        if len(saholicUserAmountMap) > 0:

            batchUpdateMap['userAmount'] = json.dumps(saholicUserAmountMap)

            batchUpdateMap['batchId'] = batchId

            request = urllib2.Request(WALLET_CREDIT_URL, headers=headers)

            data = urllib.urlencode(batchUpdateMap)

            response = urllib2.urlopen(request, data)

            return json.loads(response.read())['response']['credited']

        else:

            tprint("Nothing to Refund")

            return False

    except:

        traceback.print_exc()

        tprint("Could not batch refund")

        return False

def main():

#    client = MongoClient('mongodb://root:ecip$dtrMay2014@dtr:27017/')

#    s = Store(1)

#    orders = s.db.Dtr.find({"subOrders.imgUrl":{"$exists":1}}, {"merchantOrderId":1, "subOrders.productCode":1,"subOrders.imgUrl":1,"_id": 0})

#    db = client.Dtr

#    for order in orders:

#        for subOrder in order.get("subOrders"):

#            #db.merchantOrder.update({"merchantOrderId":order.getMerchantOrderId,"subOrders.imgUrl":{"$exists":0}, "subOrders.productCode":subOrder.get("productCode")}, {"$set":{"subOrders.$.imgUrl":subOrder.get("imgUrl")}})

#            db.merchantOrder.findOne()

    settlePayBack('dry')        

###

#Settlement process is suposed to be a batch and run weekly

#It is should be running on first hour of mondays. As cron should

# Maintain a batch id.

def getBrowserObject():

    import cookielib

    br = mechanize.Browser(factory=mechanize.RobustFactory())

    cj = cookielib.LWPCookieJar()

    br.set_cookiejar(cj)

    br.set_handle_equiv(True)

    br.set_handle_redirect(True)

    br.set_handle_referer(True)

    br.set_handle_robots(False)

    br.set_debug_http(False)

    br.set_debug_redirects(False)

    br.set_debug_responses(False)

    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)

    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),

                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),

                     ('Accept-Encoding', 'gzip,deflate,sdch'),                  

                     ('Accept-Language', 'en-US,en;q=0.8'),                     

                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]

    return br

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj

def ungzipResponse(r):

    headers = r.info()

    if headers['Content-Encoding']=='gzip':

        import gzip

        gz = gzip.GzipFile(fileobj=r, mode='rb')

        html = gz.read()

        gz.close()

        return html

def tprint(*msg):

    print datetime.now(), "-", msg

if __name__ == '__main__':

    main()