WebSVN – SmartDukaan – /trunk/PyDTR/src/dtr/main.py

'''
Created on Jan 15, 2015

@author: amit
'''
from datetime import datetime, timedelta
from dtr.config import PythonPropertyReader
from dtr.storage.Mongo import getDealRank
from dtr.storage.Mysql import getOrdersAfterDate1, getOrdersAfterDate
from pprint import pprint
from pymongo.mongo_client import MongoClient
import base64
import importlib
import json
import math
import mechanize
import time
import traceback
import urllib
import urllib2
sourceMap = {1:"amazon", 2:"flipkart", 3:"snapdeal", 4:"spice", 5:"shopclues", 6:"paytm"}
headers = { 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      
            'Accept-Language' : 'en-US,en;q=0.8',                     
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
        }
CASHBACK_URL = PythonPropertyReader.getConfig('CASHBACK_URL')
USER_LOOKUP_URL = PythonPropertyReader.getConfig('USER_LOOKUP_URL')
WALLET_CREDIT_URL = PythonPropertyReader.getConfig('WALLET_CREDIT_URL')

def getStore(source_id):
    #module = sourceMap[source_id]
    store = Store(source_id)
    try:
        module = importlib.import_module("dtr.sources." + sourceMap[source_id])
        store = getattr(module, "Store")(source_id)
        return store
    except:
        traceback.print_exc()
        return None

class ScrapeException(Exception):
    """Exception raised for errors in the input.

    Attributes:
        expr -- input expression in which the error occurred
        msg  -- explanation of the error
    """

    def __init__(self, expr, msg):
        self.expr = expr
        self.msg = msg

class ParseException(Exception):
    """Exception raised for errors in the input.

    Attributes:
        expr -- input expression in which the error occurred
        msg  -- explanation of the error
    """

    def __init__(self, expr, msg):
        self.expr = expr
        self.msg = msg

client = MongoClient('mongodb://localhost:27017/') 

class Store(object):
    
    ORDER_PLACED = 'Order Placed'
    ORDER_DELIVERED = 'Delivered'
    ORDER_SHIPPED = 'Shipped' #Lets see if we can make use of it
    ORDER_CANCELLED = 'Cancelled'
    
    CB_INIT = 'Waiting Confirmation'
    CB_PENDING = 'Pending'
    CB_CREDIT_IN_PROCESS = 'Credit in process'
    CB_CREDITED = 'Credited to wallet'
    CB_NA = 'Not Applicable'
    CB_APPROVED = 'Approved'
    CB_REJECTED = 'Rejected'
    CB_ONHOLD = 'On hold'
    CB_CANCELLED = 'Cancelled'
    
    CONF_CB_SELLING_PRICE = 0
    CONF_CB_DISCOUNTED_PRICE = 1
    
    def __init__(self, store_id):
        self.db = client.Dtr
        self.store_id = store_id
        self.store_name = sourceMap[store_id]
    
    '''
    To Settle payback for respective stores.
    Also ensures that settlement happens only for approved orders
    '''
    
    def getName(self):
        raise NotImplementedError
    
    def scrapeAffiliate(self, startDate=None, endDate=None):
        raise NotImplementedError
    
    def getTrackingUrls(self, userId):
        raise NotImplementedError
        
    
    def saveToAffiliate(self, offers):
        raise NotImplementedError
    
    def getUpdateMap(self, subOrder, cashBackStatus):
        updateMap = {}
        closed = False
        for (key,value) in subOrder.iteritems():
            if key=="status":
                if value==Store.ORDER_CANCELLED:
                    closed = True
                    updateMap['subOrders.$.closed'] = True
                    if cashBackStatus == Store.CB_PENDING:
                        updateMap['subOrders.$.cashBackStatus'] = Store.CB_CANCELLED
                if value==Store.ORDER_DELIVERED:
                    closed = True
                    updateMap['subOrders.$.closed'] = True
                    if cashBackStatus == Store.CB_PENDING:
                        updateMap['subOrders.$.cashBackStatus'] = Store.CB_APPROVED
            updateMap['subOrders.$.' + key] = value
        subOrder['closed'] = closed
        return updateMap
    
    def saveOrder(self, merchantOrder):
        #merchantOrder = Order()
        #merchantOrder.status
        #mercha
        pass
    
    
    def scrapeStoreOrders(self,):
        raise NotImplementedError
    
    def updateSubOrder(self, subOrder):
        pass
        #if subOrder.get
    
    def _saveToOrder(self, order):
        collection = self.db.merchantOrder
        try:
            order = collection.insert(order)
            return True 
        except Exception as e:
            return False

    def _updateToOrder(self, order):
        collection = self.db.merchantOrder
        try:
            print "************************************************************************"
            print "order"
            print order
            print "************************************************************************"
            collection.update({"orderId":order['orderId']},{"$set":order}, upsert = True)
            #merchantOder 
        except Exception as e:
            traceback.print_exc()
    
    def getCashbackAmount(self, productCode, amount):
        alagvar = CASHBACK_URL % (productCode,self.store_id)
        filehandle = urllib2.Request(alagvar,headers=headers)
        x= urllib2.urlopen(filehandle)
        rmap = json.loads(str(x.read()))
        if len(rmap)==0:
            return (0,0)
        else:
            if rmap['cash_back_description'] == 'PERCENTAGE':
                return (math.floor((amount * rmap['cash_back'])/100), rmap['cash_back'])
            else:
                return (rmap['cash_back'], 0)
            
    
    '''
    Parses the order for specific store
    
    order id, total amount, created on(now() if could not parse
    suborder id, title, quantity, unit price, expected delivery date,
    status (default would be Order placed)
    
    once products are identified, each suborder can then be updated
    with respective cashback.
    
    Possible fields to display for Not yet delivered orders are 
    Product/Quantity/Amount/Store/CashbackAmount/OrderDate/ExpectedDelivery/OrderStaus/DetailedStatus/CashbackStatus
    No need to show cancelled orders.
    CashbackStatus - NotApplicable/Pending/Approved/Cancelled/CreditedToWallet
    OrderStatus - Placed/Cancelled/Delivered
    '''
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        
        pass
    
    def _updateOrdersPayBackStatus(self, searchMap, updateMap):
        searchMap['subOrders.missingAff'] = False
        updateMap['subOrders.$.missingAff'] = True
        self.db.merchantOrder.update(searchMap, { '$set': updateMap }, multi=True)
        
    def _getActiveOrders(self, searchMap={}, collectionMap={}):
        collection = self.db.merchantOrder
        searchMap = dict(searchMap.items()+ {"closed": False, "storeId" : self.store_id}.items()) 
        collectionMap =  dict(collectionMap.items() + {"orderSuccessUrl":1, "orderId":1,"subOrders":1, "placedOn":1}.items())
        stores = collection.find(searchMap, collectionMap)
        return [store for store in stores]
    
    def _getMissingOrders(self,searchMap={}):
        collection = self.db.merchantOrder
        searchMap = dict(searchMap.items()+ {"requireDetail":True, "storeId":self.store_id}.items()) 
        orders = collection.find(searchMap)
        return list(orders)
    
    
    def _isSubOrderActive(self,order, merchantSubOrderId):
        subOrders = order.get("subOrders")
        if subOrders is None:
            return None
        for subOrder in subOrders:
            if merchantSubOrderId == subOrder.get("merchantSubOrderId"):
                return subOrder
        return None
    
    def populateDerivedFields(self, order):
        closed=True
        for subOrder in order.subOrders:
            if subOrder.closed:
                continue
            cashbackAmount, cashbackPercent = self.getCashbackAmount(subOrder.productCode, subOrder.amountPaid)
            cashbackStatus = Store.CB_PENDING
            if cashbackAmount <= 0:
                cashbackStatus = Store.CB_NA
            subOrder.cashBackAmount = cashbackAmount
            subOrder.cashBackPercentage = cashbackPercent
            subOrder.cashBackStatus = cashbackStatus      
            subOrder.closed = subOrder.status in [self.ORDER_CANCELLED, self.ORDER_DELIVERED]
            closed = closed and subOrder.closed
            dealRank = getDealRank(subOrder.productCode, self.store_id, order.userId)
            subOrder.dealRank = dealRank.get('rank')
            subOrder.rankDesc = dealRank.get('description')
            subOrder.maxNlc = dealRank.get('maxNlc')
            subOrder.minNlc = dealRank.get('minNlc')
            subOrder.db = dealRank.get('dp')
            subOrder.itemStatus = dealRank.get('status')
        order.closed = closed
    
def settlePayBack(runtype='dry'):
    userAmountMap = {}
    searchMapList = []
    for mo in client.Dtr.merchantOrder.find({"subOrders.cashBackStatus":Store.CB_APPROVED}):
        userId = mo.get("userId")
        if   mo.get('subOrders') is not None:
            for so in mo['subOrders']:
                if so.get('cashBackStatus') == Store.CB_APPROVED:
                    searchMapList.append({"orderId":mo.get("orderId"), "subOrders.merchantSubOrderId":so.get("merchantSubOrderId")})
                    if not userAmountMap.has_key(userId):
                        userAmountMap[userId] = so.get('cashBackAmount')
                    else:
                        userAmountMap[userId] += so.get('cashBackAmount')
                    print "%s\t%s\t%s\t%s\t%s\t%s\t%s"%(userId, mo.get("orderId"), so.get("merchantSubOrderId"),so.get("productTitle") ,so.get("cashBackStatus"), so.get("cashBackAmount"), so.get("batchId"))
                else:  
                    print "%s\t%s\t%s\t%s\t%s\t%s\t%s"%(userId, mo.get("orderId"), so.get("merchantSubOrderId"),so.get("productTitle") ,so.get("cashBackStatus"), so.get("cashBackAmount"), so.get("batchId"))
    for searchMap in searchMapList:
        print "%s\t%s"%(searchMap.get('orderId'),searchMap.get('subOrders.merchantSubOrderId'))
    for key,val in userAmountMap.iteritems():
        print "%s\t%s"%(key,val)
    if (runtype=='live'):
        bulk = client.Dtr.merchantOrder.initialize_ordered_bulk_op()
        if len(searchMapList) == 0:
            return
        for searchMap in searchMapList:
            bulk.find(searchMap).update({'$set' : {'subOrders.$.cashBackStatus':Store.CB_CREDIT_IN_PROCESS}})
        bulk.execute()

        datetimeNow = datetime.now() 
        batchId = int(time.mktime(datetimeNow.timetuple()))
        if refundToWallet(batchId, userAmountMap):
            bulk = client.Dtr.merchantOrder.initialize_ordered_bulk_op()
            for searchMap in searchMapList:
                bulk.find(searchMap).update({'$set' : {'subOrders.$.cashBackStatus':Store.CB_CREDITED, "subOrders.$.batchId":batchId}})
            print bulk.execute()
            for key, value in userAmountMap.iteritems():
                client.Dtr.refund.insert({"userId": key, "batch":batchId, "userAmount":value, "timestamp":datetime.strftime(datetimeNow,"%Y-%m-%d %H:%M:%S")})
                client.Dtr.user.update({"userId":key}, {'$inc': { "credited": value}}, upsert=True)
            tprint("PayBack Settled")
        else:
            tprint("Error Occurred while running batch. Rolling Back")
            bulk = client.Dtr.merchantOrder.initialize_ordered_bulk_op()
            for searchMap in searchMapList:
                bulk.find(searchMap).update({'$set' : {'subOrders.$.cashBackStatus':Store.CB_APPROVED}})
            bulk.execute()


def settlePayBack1(runtype='dry'):
        approvedUserMap = {}
        pendingUserMap = {}
        creditedToWalletUserMap = {}
        #client.Dtr.merchantOrder.update({'subOrders.cashBackStatus':Store.CB_APPROVED},{'$set':{'subOrders.$.cashBackStatus':Store.CB_CREDIT_IN_PROCESS}}, multi=True)    
        for mo in client.Dtr.merchantOrder.find({"subOrders.cashBackStatus":Store.CB_CREDITED}):
            userId = mo.get("userId")
            for so in mo['subOrders']:
                print "%s\t%s\t%s\t%s\t%s\t%s\t%s"%(userId, mo.get("orderId"), so.get("merchantSubOrderId"),so.get("productTitle") ,so.get("cashBackStatus"), so.get("cashBackAmount"), so.get("batchId"))

        for refund in client.Dtr.refund.find():
            print "%s\t%s\t%s\t%s"%(refund.get("userId") ,refund.get("timestamp") ,refund.get("userAmount"), refund.get("batch"))


def refundToWallet(batchId, userAmountMap):
    base64string = base64.encodestring('%s:%s' % ("dtr", "dtr18Feb2015")).replace('\n', '')
    batchUpdateMap = {}
    try :
        saholicUserAmountMap = {}
        for key, value in userAmountMap.iteritems():
            userLookupRequest = urllib2.Request(USER_LOOKUP_URL %(key), headers=headers)
            userLookupRequest.add_header("Authorization", "Basic %s" % base64string)
            try:
                response = urllib2.urlopen(userLookupRequest).read()
                saholicUserId = json.loads(response)['account_key']
                saholicUserAmountMap[saholicUserId] = value
            except:
                tprint("Could not fetch saholic id for user : " + str(key))
                continue
        if len(saholicUserAmountMap) > 0:
            batchUpdateMap['userAmount'] = json.dumps(saholicUserAmountMap)
            batchUpdateMap['batchId'] = batchId
            request = urllib2.Request(WALLET_CREDIT_URL, headers=headers)
            data = urllib.urlencode(batchUpdateMap)
            response = urllib2.urlopen(request, data)
            return json.loads(response.read())['response']['credited']
        else:
            tprint("Nothing to Refund")
            return False
    except:
        traceback.print_exc()
        tprint("Could not batch refund")
        return False
    
def main():
#    client = MongoClient('mongodb://root:ecip$dtrMay2014@dtr:27017/')
#    s = Store(1)
#    orders = s.db.Dtr.find({"subOrders.imgUrl":{"$exists":1}}, {"merchantOrderId":1, "subOrders.productCode":1,"subOrders.imgUrl":1,"_id": 0})
#    db = client.Dtr
#    for order in orders:
#        for subOrder in order.get("subOrders"):
#            #db.merchantOrder.update({"merchantOrderId":order.getMerchantOrderId,"subOrders.imgUrl":{"$exists":0}, "subOrders.productCode":subOrder.get("productCode")}, {"$set":{"subOrders.$.imgUrl":subOrder.get("imgUrl")}})
#            db.merchantOrder.findOne()
    settlePayBack('dry')        


###
#Settlement process is suposed to be a batch and run weekly
#It is should be running on first hour of mondays. As cron should
# Maintain a batch id.


def getBrowserObject():
    import cookielib
    br = mechanize.Browser(factory=mechanize.RobustFactory())
    cj = cookielib.LWPCookieJar()
    br.set_cookiejar(cj)
    br.set_handle_equiv(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)
    br.set_debug_http(False)
    br.set_debug_redirects(False)
    br.set_debug_responses(False)
    
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    
    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),
                     ('Accept-Encoding', 'gzip,deflate,sdch'),                  
                     ('Accept-Language', 'en-US,en;q=0.8'),                     
                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]
    return br
def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj

def ungzipResponse(r):
    headers = r.info()
    if headers['Content-Encoding']=='gzip':
        import gzip
        gz = gzip.GzipFile(fileobj=r, mode='rb')
        html = gz.read()
        gz.close()
        return html

def tprint(*msg):
    print datetime.now(), "-", msg
    
if __name__ == '__main__':
    main()
Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/main.py – Rev 16419
