Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from bson.binary import Binary

from datetime import datetime, date, timedelta

from dtr import main

from dtr.config import PythonPropertyReader

from dtr.dao import AffiliateInfo, Order, SubOrder

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \

    Store as MStore, ungzipResponse, tprint

from pprint import pprint

from pymongo import MongoClient

import json

import pymongo

import re

import traceback

import urllib

import urllib2

from dtr.storage.Mongo import getDealRank

ORDERSTATUS  = {"PAYMENT_PENDING" : 0,

"PAYMENT_FAILED" : 1,

"COD_VERIFICATION_PENDING" : 2,

"SUBMITTED_FOR_PROCESSING" : 3,

"ACCEPTED" : 4,

"INVENTORY_LOW" : 5,

"REJECTED" : 6,

"BILLED" : 7,

"PAYMENT_FLAGGED" : 8,

"SHIPPED_FROM_WH" : 9,

"SHIPPED_TO_LOGST" : 10,

"PAYMENT_FLAGGED_DENIED" : 11,

"DELIVERY_SUCCESS" : 12,

"CANCEL_REQUEST_RECEIVED" : 13,

"CANCEL_REQUEST_CONFIRMED" : 14,

"CANCELLED_ON_CUSTOMER_REQUEST" : 15,

"SHIPPED_TO_DESTINATION_CITY" : 16,

"REACHED_DESTINATION_CITY" : 17,

"COD_VERIFICATION_FAILED" : 18,

"FAILED" : 19,

"RTO_IN_TRANSIT" : 20,

"RTO_RECEIVED_PRESTINE" : 21,

"DOA_PICKUP_REQUEST_RAISED" : 22,

"DOA_PICKUP_CONFIRMED" : 23,

"DOA_RETURN_IN_TRANSIT" : 24,

"DOA_RECEIVED_PRESTINE" : 25,

"DOA_CERT_INVALID" : 26,

"DOA_CERT_VALID" : 27,

"RTO_RESHIPPED" : 28,

"DOA_INVALID_RESHIPPED" : 29,

"DOA_VALID_RESHIPPED" : 30,

"RTO_REFUNDED" : 31,

"DOA_VALID_REFUNDED" : 32,

"DOA_INVALID_REFUNDED" : 33,

"CANCELLED_DUE_TO_LOW_INVENTORY" : 34,

"LOW_INV_PO_RAISED" : 35,

"LOW_INV_REVERSAL_IN_PROCESS" : 36,

"LOW_INV_NOT_AVAILABLE_AT_HOTSPOT" : 37,

"LOW_INV_PO_RAISED_TIMEOUT" : 38,

"LOW_INV_REVERSAL_TIMEOUT" : 39,

"FIRST_DELIVERY_ATTEMPT_MADE" : 40,

"CAPTURE_IN_PROCESS" : 41,

"DOA_REQUEST_RECEIVED" : 42,

"DOA_REQUEST_AUTHORIZED" : 43,

"DOA_PICKUP_DENIED" : 44,

"DOA_RECEIVED_DAMAGED" : 45,

"DOA_LOST_IN_TRANSIT" : 46,

"DOA_RESHIPPED_RCVD_DAMAGED" : 47,

"DOA_REFUNDED_RCVD_DAMAGED" : 48,

"DOA_RESHIPPED_LOST_IN_TRANSIT" : 49,

"DOA_REFUNDED_LOST_IN_TRANSIT" : 50,

"RTO_RECEIVED_DAMAGED" : 51,

"RTO_LOST_IN_TRANSIT" : 52,

"RTO_DAMAGED_RESHIPPED" : 53,

"RTO_DAMAGED_REFUNDED" : 54,

"RTO_LOST_IN_TRANSIT_RESHIPPED" : 55,

"RTO_LOST_IN_TRANSIT_REFUNDED" : 56,

"RTO_INVENTORY_REVERSED" : 57,

"RET_REQUEST_RECEIVED" : 58,

"RET_REQUEST_AUTHORIZED" : 59,

"RET_PICKUP_REQUEST_RAISED" : 60,

"RET_PICKUP_DENIED" : 61,

"RET_PICKUP_CONFIRMED" : 62,

"RET_RETURN_IN_TRANSIT" : 63,

"RET_RECEIVED_PRESTINE" : 64,

"RET_RECEIVED_DAMAGED" : 65,

"RET_LOST_IN_TRANSIT" : 66,

"RET_PRODUCT_USABLE" : 67,

"RET_PRODUCT_UNUSABLE" : 68,

"RET_PRODUCT_USABLE_RESHIPPED" : 69,

"RET_PRODUCT_USABLE_REFUNDED" : 70,

"RET_PRODUCT_UNUSABLE_RESHIPPED" : 71,

"RET_PRODUCT_UNUSABLE_REFUNDED" : 72,

"RET_RESHIPPED_RCVD_DAMAGED" : 73,

"RET_REFUNDED_RCVD_DAMAGED" : 74,

"RET_RESHIPPED_LOST_IN_TRANSIT" : 75,

"RET_REFUNDED_LOST_IN_TRANSIT" : 76,

"LOST_IN_TRANSIT" : 77,

"LOST_IN_TRANSIT_RESHIPPED" : 78,

"LOST_IN_TRANSIT_REFUNDED" : 79,

"DELIVERED_AT_STORE" : 80,

"RECEIVED_AT_STORE" : 81}

headers = { 

           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'

        }

SAHOLIC_ORDER_URL=PythonPropertyReader.getConfig('SAHOLIC_ORDER_URL')

SAHOLIC_ORDER_URL_TR=PythonPropertyReader.getConfig('SAHOLIC_ORDER_URL_TR')

class Store(MStore):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      MStore.ORDER_PLACED : [2,3,4,5,6,7,8,13],

                      MStore.ORDER_DELIVERED : [12],

                      MStore.ORDER_SHIPPED : [9,10,16,17,81,80, 40,20],

                      MStore.ORDER_CANCELLED : [1,6,11,14,15,18,19,31,34,28,53,54,55,56,20,21,77,79, 78]

                      }

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp = {}

        paymentId = int(re.findall(r'=(.*)?',orderSuccessUrl)[0])

        orderRequest = urllib2.Request(SAHOLIC_ORDER_URL %(paymentId), headers=headers)

        try:

            self.userId = userId

            connection = urllib2.urlopen(orderRequest)

            response = connection.read()

            connection.close()

            response = json.loads(response)['response']

            payment = response['payment']

            subOrders = []

            orders = response['orders']

            items = response['itemsMap']

            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

            merchantOrder.merchantOrderId = payment['merchantTxnId']

            merchantOrder.paidAmount = payment['amount']

            for o in orders:

                subOrders.append(self.createSubOrder(o, items))

            merchantOrder.subOrders = subOrders

            merchantOrder.placedOn = merchantOrder.subOrders[0].placedOn

            s = todict(merchantOrder)

            s['sUserId'] = orders[0]['customer_id']

            if response.get('payment_option') is not None:

                s['payment_option'] = response ['payment_option']

            if self._saveToOrder(s):

                resp['result'] = 'ORDER_CREATED'

            else:

                #Order already created

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

        except:

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

        return resp

    def createSubOrder(self, order, items):

            lineitem = order['lineitems'][0]

            item = items[str(lineitem['item_id'])]

            brand = lineitem.get('brand')

            modelNumber = lineitem.get('model_number')

            modelName = lineitem.get('model_name')

            color = lineitem.get('color')

            productTitle = brand + (" " + modelName if modelName else "") + (" "  + modelNumber if modelNumber is not None else "") + ("(" + color +")" if color else "")

            amountPaid = order['total_amount'] 

            subOrder = SubOrder(productTitle, None, datetime.strftime(datetime.fromtimestamp(order['created_timestamp']/1000),"%d %B %Y"), amountPaid)

            subOrder.amount = amountPaid - order['gvAmount']

            subOrder.merchantSubOrderId = str(order['id'])

            subOrder.orderDetailUrl = "http://m.saholic.com/order/" + subOrder.merchantSubOrderId 

            subOrder.quantity = int(lineitem['quantity'])

            subOrder.estimatedDeliveryDate = datetime.strftime(datetime.fromtimestamp(order['promised_delivery_time']/1000),"%d %B %Y")

            subOrder.imgUrl = item.get("imgUrl") 

            subOrder.productUrl = "http://m.saholic.com/" + item['url']

            subOrder.productCode = item['url'].split('-')[-1]

            subOrder.detailedStatus = order['statusDescription']

            (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amountPaid/subOrder.quantity)

            dealRank = getDealRank(subOrder.productCode, self.store_id, self.userId)

            subOrder.dealRank = dealRank.get('rank')

            subOrder.rankDesc = dealRank.get('description')

            subOrder.maxNlc = dealRank.get('maxNlc')

            subOrder.minNlc = dealRank.get('minNlc')

            subOrder.db = dealRank.get('dp')

            subOrder.itemStatus = dealRank.get('status')

            cashbackStatus = Store.CB_PENDING

            if cashbackAmount <= 0:

                cashbackStatus = Store.CB_NA

            subOrder.cashBackStatus = cashbackStatus

            subOrder.cashBackAmount = cashbackAmount*subOrder.quantity

            if percentage > 0:

                subOrder.cashBackPercentage = percentage

            return subOrder

    def _getStatusFromDetailedStatus(self, detailedStatus):

        print str(detailedStatus)

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + str(detailedStatus))

    def scrapeStoreOrders(self,):

        trs = self._getActiveOrders(collectionMap={'merchantOrderId':1, 'sUserId':1, 'userId':1})

        bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

        for tr in trs:

            self.userId = tr['userId']

            orderRequest = urllib2.Request(SAHOLIC_ORDER_URL_TR %(tr['merchantOrderId'], tr['sUserId']), headers=headers)

            try:

                response = urllib2.urlopen(orderRequest).read()

                print "transaction_id----------", tr['merchantOrderId']

                response = json.loads(response)['response']

                items = response['itemsMap']

                orders = response['orders']

                closed = True

                splitSubOrdersMap={}

                ordersToClone = {}

                for order in orders:

                    #print "orderid---", order['id']

                    orderId = str(order['id'])

                    subOrder = self._isSubOrderActive(tr, orderId)

                    currentQty = int(order['lineitems'][0]['quantity'])

                    if subOrder:

                        if subOrder['closed']:

                            continue

                        else:

                            quantity = int(subOrder['quantity'])

                            findMap = {"orderId": tr['orderId'], "subOrders.merchantSubOrderId": orderId}

                            updateMap = {}

                            updateMap["subOrders.$.detailedStatus"] = order['statusDescription']

                            status = self._getStatusFromDetailedStatus(ORDERSTATUS[order['status']]) 

                            closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                            updateMap["subOrders.$.status"] = status

                            #Check if split

                            if quantity != currentQty:

                                updateMap["subOrders.$.quantity"] = currentQty

                                updateMap["subOrders.$.amount"] = int((subOrder['amount']*currentQty)/quantity)

                                updateMap["subOrders.$.cashBackAmount"] = int((subOrder['cashBackAmount']*currentQty)/quantity)

                                updateMap["subOrders.$.amountPaid"] = int((subOrder['amountPaid']*currentQty)/quantity)

                                splitSubOrdersMap[order['id']]=subOrder

                            if closedStatus:

                                #if status is closed then change the paybackStatus accordingly

                                updateMap["subOrders.$.closed"] = True

                                if status == Store.ORDER_DELIVERED:

                                    deliveredOn = datetime.strftime(datetime.fromtimestamp(order['delivery_timestamp']/1000),"%A %d %B %Y")

                                    updateMap['subOrders.$.deliveredOn'] = deliveredOn

                                    if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                        updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                                elif status == Store.ORDER_CANCELLED:

                                    if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                        updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                            else:

                                expectedDelivery = datetime.strftime(datetime.fromtimestamp(order['promised_delivery_time']/1000),"%A %d %B %Y")

                                updateMap["subOrders.$.estimatedDeliveryDate"] = expectedDelivery

                                closed = False

                            bulk.find(findMap).update({'$set' : updateMap})

                    else:

                        if order['originalOrderId']:

                            ordersToClone[order['id']] = order

                        else:

                            subOrder = self.createSubOrder(order, items)

                            self.db.merchantOrder.update({"orderId":tr['orderId']},{'$push':{"subOrders":todict(subOrder)}})

                            print "Added new suborder with subOrder Id:", subOrder.merchantSubOrderId

                            closed = False

                for order in ordersToClone.values():

                    originalOrderId = self.getOriginalOrderId(order['id'], ordersToClone, splitSubOrdersMap)

                    subOrderToClone = splitSubOrdersMap[originalOrderId].copy()

                    subOrderToClone["merchantSubOrderId"] = str(order['id'])

                    currentQty = int(order['lineitems'][0]['quantity'])

                    quantity = int(subOrderToClone['quantity'])

                    subOrderToClone["detailedStatus"] = order['statusDescription']

                    status = self._getStatusFromDetailedStatus(ORDERSTATUS[order['status']]) 

                    closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                    print "status---", status, order['status']

                    subOrderToClone["status"] = status

                    subOrderToClone["quantity"] = currentQty

                    subOrderToClone["amount"] = int((subOrderToClone['amount']*currentQty)/quantity)

                    subOrderToClone["cashBackAmount"] = int((subOrderToClone['cashBackAmount']*currentQty)/quantity)

                    subOrderToClone["amountPaid"] = int((subOrderToClone['amountPaid']*currentQty)/quantity)

                    if closedStatus:

                        #if status is closed then change the paybackStatus accordingly

                        subOrderToClone["closed"] = True

                        if status == Store.ORDER_DELIVERED:

                            deliveredOn = datetime.strftime(datetime.fromtimestamp(order['delivery_timestamp']/1000),"%A %d %B %Y")

                            subOrderToClone['deliveredOn'] = deliveredOn

                            if subOrderToClone.get("cashBackStatus") == Store.CB_PENDING:

                                subOrderToClone["cashBackStatus"] = Store.CB_APPROVED

                        elif status == Store.ORDER_CANCELLED:

                            if subOrderToClone.get("cashBackStatus") == Store.CB_PENDING:

                                subOrderToClone["cashBackStatus"] = Store.CB_CANCELLED

                    else:

                        expectedDelivery = datetime.strftime(datetime.fromtimestamp(order['promised_delivery_time']/1000),"%A %d %B %Y")

                        subOrderToClone["estimatedDeliveryDate"] = expectedDelivery

                        closed = False

                    self.db.merchantOrder.update({"orderId":tr['orderId']},{'$push':{"subOrders":subOrderToClone}})

                bulk.find({"orderId":tr['orderId']}).update({'$set' : {'closed':closed}})

            except:

                print "something went wrong for request", orderRequest

                traceback.print_exc()

        try:

            bulk.execute()

        except:

            print "Could not execute bulk"

            traceback.print_exc()

    def getOriginalOrderId(self, orderId, ordersToClone, splitSubOrdersMap): 

        if splitSubOrdersMap.has_key(orderId):

            return orderId

        else:

            originalOrderId = ordersToClone[orderId]['originalOrderId']

            return self.getOriginalOrderId(originalOrderId, ordersToClone, splitSubOrdersMap) 

    def _saveToAffiliate(self, offers):

        if offers is None or len(offers)==0:

            print "no affiliate have been pushed"

            return

        collection = self.db.snapdealOrderAffiliateInfo

        try:

            collection.insert(offers,continue_on_error=True)

        except pymongo.errors.DuplicateKeyError as e:

            print e.details

    def covertToObj(self,offer):

        offerData = offer['Stat']

        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 

                              offerData['datetime'], offerData['payout'], offer['Offer']['name'], offerData['ip'], offerData['conversion_sale_amount'])

        return offer1

def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):

    endDate=date.today() + timedelta(days=1)

    startDate=endDate - timedelta(days=31)

    parameters = (

        ("page",str(page)),

        ("limit",str(limit)),

        ("fields[]","Stat.offer_id"),

        ("fields[]","Stat.datetime"),

        ("fields[]","Offer.name"),

        ("fields[]","Stat.conversion_status"),

        ("fields[]","Stat.conversion_sale_amount"),

        ("fields[]","Stat.payout"),

        ("fields[]","Stat.ip"),

        ("fields[]","Stat.ad_id"),

        ("fields[]","Stat.affiliate_info1"),

        ("sort[Stat.datetime]","desc"),

        ("filters[Stat.date][conditional]","BETWEEN"),

        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),

        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),

        ("data_start",startDate.strftime('%Y-%m-%d')),

        ("data_end",endDate.strftime('%Y-%m-%d')),

        ("Method","getConversions"),

        ("NetworkId","jasper"),

        ("SessionToken",token),

    )

    #Encode the parameters

    return urllib.urlencode(parameters)

def main():

    store = getStore(4)

    store.scrapeStoreOrders()

    #store._isSubOrderActive(8, "5970688907")

    #store.scrapeAffiliate()

    #store.parseOrderRawHtml(112345, "subtagId", 1122323,  "html", 'http://www.saholic.com/pay-success?paymentId=1798123')

    #print store.getCashbackAmount('1011378', 500)

if __name__ == '__main__':

    main()

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj