Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from bson.binary import Binary

from datetime import datetime, date

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, Store as MStore

from pprint import pprint

from pymongo import MongoClient

import json

import pymongo

import re

import traceback

import urllib

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

AFFILIATE_URL='http://affiliate.snapdeal.com'

POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'

ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'

CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'

class Store(MStore):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      MStore.ORDER_PLACED : ['In Progress','N/A'],

                      MStore.ORDER_DELIVERED : ['Delivered'],

                      MStore.ORDER_SHIPPED : ['In Transit'],

                      MStore.ORDER_CANCELLED : ['Closed For Vendor Reallocation', 'Cancelled']

                      }

    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def getName(self):

        return "snapdeal"

    def scrapeAffiliate(self, startDate=None, endDate=None):

        br = getBrowserObject()

        br.open(AFFILIATE_URL)

        br.select_form(nr=0)

        br.form['data[User][password]'] = PASSWORD 

        br.form['data[User][email]'] = USERNAME

        br.submit()

        response = br.open(CONFIG_URL)

        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response, br), re.IGNORECASE)[0]

        allOffers = self._getAllOffers(br, token)

        allPyOffers = []

        maxSaleDate = self._getLastSaleDate()

        newMaxSaleDate = maxSaleDate

        for offer in allOffers:

            allPyOffers.append(self.covertToObj(offer).__dict__)

            saleDate = datetime.strptime(offer.saleDate,"%Y-%m-%d %H:%M:%S")

            if maxSaleDate < saleDate:

                self._updateOrdersPayBackStatus(offer.subTagId,offer.saleDate)

                if newMaxSaleDate < saleDate:

                    newMaxSaleDate = saleDate

        self._setLastSaleDate(newMaxSaleDate)

        print allPyOffers

        self._saveToAffiliate(allPyOffers)

    def _setLastSaleDate(self, saleDate):

        self.db.lastDaySale.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})

    def _updateOrdersPayBackStatus(self, subTagId, saleDate):

        self.db.merchantOrder.update({"subTagId":subTagId, "placedOn":saleDate, "subOrders.cashbackStatus":MStore.CB_INIT},

                                              { '$set': { "subOrders.$.cashbackStatus" : MStore.CB_PENDING } })

    def _getLastSaleDate(self,):

        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})

        if lastDaySaleObj is None:

            return datetime.min

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        try:

            br = getBrowserObject()

            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

            response = br.open(url)

            page = ungzipResponse(response, br)

            #page=page.decode("utf-8")

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            #orderHead = soup.find(name, attrs, recursive, text)

            sections = soup.findAll("section")

            #print sections

            order = sections[1]

            orderTrs = order.findAll("tr")

            placedOn = str(orderTrs[0].findAll("td")[1].text)

            #Pop two section elements

            sections.pop(0) 

            sections.pop(0)

            subOrders = sections

            merchantSubOrders = []

            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

            merchantOrder.merchantOderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]

            for orderTr in orderTrs:

                orderTrString = str(orderTr)

                if "Total Amount" in orderTrString:

                    merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]

                elif "Delivery Charges" in orderTrString:

                    merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]

                elif "Discount Applied" in orderTrString:

                    merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

                elif "Paid Amount" in orderTrString:

                    merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]

            for subOrderElement in subOrders:

                productUrl = str(subOrderElement.find("a")['href'])

                subTable = subOrderElement.find("table", {"class":"lrPad"})

                subTrs = subTable.findAll("tr")

                unitPrice=None

                offerDiscount = None

                deliveryCharges = None

                amountPaid = None

                for subTr in subTrs:

                    subTrString = str(subTr)

                    if "Unit Price" in subTrString:

                        unitPrice = re.findall(r'\d+', subTrString)[0]

                    if "Quantity" in subTrString:

                        qty = re.findall(r'\d+', subTrString)[0]

                    elif "Offer Discount" in subTrString:

                        offerDiscount =   re.findall(r'\d+', subTrString)[0]

                    elif "Delivery Charges" in subTrString:

                        deliveryCharges =   re.findall(r'\d+', subTrString)[0]

                    elif "Subtotal" in subTrString:

                        if int(qty) > 0:

                            amountPaid =   str(int(re.findall(r'\d+', subTrString)[0])/int(qty))

                        else:

                            amountPaid =   "0"

                if self.CONF_CB_AMOUNT == MStore.CONF_CB_SELLING_PRICE or offerDiscount is None:

                    amount = int(unitPrice)

                else:

                    amount = int(unitPrice) - int(offerDiscount)

                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})

                if len(divs)<=0:

                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

                for div in divs:

                    productTitle = str(subOrderElement.find("a").text)

                    productUrl = "http://m.snapdeal.com/" + productUrl 

                    subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

                    subOrder.amountPaid = amountPaid

                    subOrder.deliveryCharges = deliveryCharges

                    subOrder.offerDiscount = offerDiscount

                    subOrder.unitPrice = unitPrice

                    subOrder.productCode = re.findall(r'\d+$', productUrl)[0]

                    cashbackAmount = self.getCashbackAmount(subOrder.productCode, amount)

                    cashbackStatus = Store.CB_INIT

                    if cashbackAmount <= 0:

                        cashbackStatus = Store.CB_NA

                    subOrder.cashBackStatus = cashbackStatus

                    subOrder.cashBackAmount = cashbackAmount

                    trackAnchor = div.find("a")   

                    if trackAnchor is not None:

                        subOrder.tracingkUrl = str(trackAnchor['href'])

                    divStr = str(div)

                    divStr = divStr.replace("\n","").replace("\t", "")

                    for line in divStr.split("<br />"):

                        if "Suborder ID" in line:

                            subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]   

                        elif "Status" in line:

                            subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                        elif "Est. Shipping Date" in line:

                            subOrder.estimatedShippingDate = line.split(":")[1].strip()

                        elif "Est. Delivery Date" in line:

                            subOrder.estimatedDeliveryDate = line.split(":")[1].strip()

                        elif "Courier Name" in line:

                            subOrder.courierName = line.split(":")[1].strip()

                        elif "Tracking No" in line:

                            subOrder.trackingNumber = line.split(":")[1].strip()

                merchantSubOrders.append(subOrder)   

            merchantOrder.subOrders = merchantSubOrders

            #print merchantOrder

            self._saveToOrder(todict(merchantOrder))

            return True

        except:

            print "Error occurred"

            traceback.print_exc()

        return False

        #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)

        #soup.find(name, attrs, recursive, text)

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus in value:

                return key

            print "Detailed Status need to be mapped"

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)

    def scrapeStoreOrders(self,):

        orders = self._getActiveOrders()

        br = getBrowserObject()

        for order in orders:

            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]

            response = br.open(url)

            page = ungzipResponse(response, br)

            #page=page.decode("utf-8")

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            sections = soup.findAll("section")

            sections.pop(0)

            sections.pop(0)

            subOrders = sections

            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

            for subOrderElement in subOrders:

                closed = True

                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})

                if len(divs)<=0:

                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

                for div in divs:

                    divStr = str(div)

                    divStr = divStr.replace("\n","").replace("\t", "")

                    updateMap = {}

                    for line in divStr.split("<br />"):

                        if "Suborder ID" in line:

                            merchantSubOrderId = re.findall(r'\d+', line)[0]

                            #break if suborder is inactive   

                            findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                            if not self._isSubOrderActive(order, merchantSubOrderId):

                                break

                        elif "Status" in line:

                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                            updateMap["subOrders.$.detailedStatus"] = detailedStatus

                            status = self._getStatusFromDetailedStatus(detailedStatus) 

                            closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                            if closedStatus:

                                #if status is closed then change the paybackStatus accordingly

                                if closedStatus == Store.ORDER_DELIVERED:

                                    if order.get("cashBackStatus") == Store.CB_PENDING:

                                        updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                                elif closedStatus == Store.ORDER_CANCELLED:

                                    if order.get("cashBackStatus") == Store.CB_PENDING:

                                        updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                            else:

                                closed = False

                        elif "Est. Shipping Date" in line:

                            estimatedShippingDate = line.split(":")[1].strip()

                            updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate

                        elif "Est. Delivery Date" in line:

                            estimatedDeliveryDate = line.split(":")[1].strip()

                            updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate

                        elif "Courier Name" in line:

                            courierName = line.split(":")[1].strip()

                            updateMap["subOrders.$.courierName"] = courierName

                        elif "Tracking No" in line:

                            trackingNumber = line.split(":")[1].strip()

                            updateMap["subOrders.$.trackingNumber"] = trackingNumber

                    bulk.find(findMap).update({'$set' : updateMap})

                    bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed}})

            result = bulk.execute()

            pprint(result)        

    def _isSubOrderActive(self,order, merchantSubOrderId):

        subOrders = order.get("subOrders")

        for subOrder in subOrders:

            if merchantSubOrderId == subOrder.get("merchantSubOrderId"):

                return True

        return False

    def _saveToAffiliate(self, offers):

        collection = self.db.snapdealOrderAffiliateInfo

        try:

            collection.insert(offers,continue_on_error=True)

        except pymongo.errors.DuplicateKeyError as e:

            print e.details

    def _saveToOrder(self, order):

        collection = self.db.merchantOrder

        try:

            order = collection.insert(order)

            #merchantOder 

        except Exception as e:

            traceback.print_exc()

    def _getAllOffers(self, br, token):

        allOffers = []

        nextPage = 1  

        while True:

            data = getPostData(token, nextPage)

            response = br.open(POST_URL, data)

            rmap = json.loads(ungzipResponse(response, br))

            if rmap is not None:

                rmap = rmap['response']

                if rmap is not None and len(rmap['errors'])==0:

                    allOffers += rmap['data']['data']

            nextPage += 1

            if rmap['data']['pageCount']<nextPage:

                break

        return allOffers

    def covertToObj(self,offer):

        offerData = offer['Stat']

        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 

                              offerData['datetime'], offerData['payout'], offer['Offer']['name'], offerData['ip'], offerData['conversion_sale_amount'])

        return offer1

def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):

    endDate=date.today() + datetime.timedelta(days=1)

    startDate=endDate - datetime.timedelta(days=31)

    parameters = (

        ("page",str(page)),

        ("limit",str(limit)),

        ("fields[]","Stat.offer_id"),

        ("fields[]","Stat.datetime"),

        ("fields[]","Offer.name"),

        ("fields[]","Stat.conversion_status"),

        ("fields[]","Stat.conversion_sale_amount"),

        ("fields[]","Stat.payout"),

        ("fields[]","Stat.ip"),

        ("fields[]","Stat.ad_id"),

        ("fields[]","Stat.affiliate_info1"),

        ("sort[Stat.datetime]","desc"),

        ("filters[Stat.date][conditional]","BETWEEN"),

        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),

        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),

        ("data_start",startDate.strftime('%Y-%m-%d')),

        ("data_end",endDate.strftime('%Y-%m-%d')),

        ("Method","getConversions"),

        ("NetworkId","jasper"),

        ("SessionToken",token),

    )

    #Encode the parameters

    return urllib.urlencode(parameters)

def main():

    store = getStore(3)

    #store._isSubOrderActive(8, "5970688907")

    store.scrapeAffiliate()

    #store.parseOrderRawHtml(12345, "subtagId", 122323,  "html", 'https://m.snapdeal.com/purchaseMobileComplete?code=1f4166d13ea799b65aa9dea68b3e9e70&order=4509499363')

def ungzipResponse(r,b):

    headers = r.info()

    if headers['Content-Encoding']=='gzip':

        import gzip

        print "********************"

        print "Deflating gzip response"

        print "********************"

        gz = gzip.GzipFile(fileobj=r, mode='rb')

        html = gz.read()

        gz.close()

        return html

if __name__ == '__main__':

    main()

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj