Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from bson.binary import Binary

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException

from pymongo import MongoClient

import datetime

import json

import pymongo

import re

import urllib

import traceback

from pprint import pprint

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

AFFILIATE_URL='http://affiliate.snapdeal.com'

POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'

ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'

CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'

class Store(main.Store):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      main.Store.ORDER_PLACED : ['In Progress','N/A'],

                      main.Store.ORDER_DELIVERED : ['Delivered'],

                      main.Store.ORDER_SHIPPED : ['In Transit'],

                      main.Store.ORDER_CANCELLED : ['Closed For Vendor Reallocation']

                      }

    def __init__(self,store_id):

        client = MongoClient('mongodb://localhost:27017/')

        self.db = client.Dtr

        super(Store, self).__init__(store_id)

    def getName(self):

        return "snapdeal"

    def scrapeAffiliate(self, startDate=None, endDate=None):

        br = getBrowserObject()

        br.open(AFFILIATE_URL)

        br.select_form(nr=0)

        br.form['data[User][password]'] = PASSWORD 

        br.form['data[User][email]'] = USERNAME

        br.submit()

        response = br.open(CONFIG_URL)

        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response, br), re.IGNORECASE)[0]

        allOffers = self._getAllOffers(br, token)

        allPyOffers = [self.covertToObj(offer).__dict__ for offer in allOffers]

        self._saveToAffiliate(allPyOffers)

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        try:

            br = getBrowserObject()

            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

            response = br.open(url)

            page = ungzipResponse(response, br)

            #page=page.decode("utf-8")

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            #orderHead = soup.find(name, attrs, recursive, text)

            sections = soup.findAll("section")

            #print sections

            order = sections[1]

            orderTrs = order.findAll("tr")

            placedOn = str(orderTrs[0].findAll("td")[1].text)

            #Pop two section elements

            sections.pop(0) 

            sections.pop(0)

            subOrders = sections

            merchantSubOrders = []

            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

            merchantOrder.merchantOderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]

            for orderTr in orderTrs:

                orderTrString = str(orderTr)

                if "Total Amount" in orderTrString:

                    merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]

                elif "Delivery Charges" in orderTrString:

                    merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]

                elif "Discount Applied" in orderTrString:

                    merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

                elif "Paid Amount" in orderTrString:

                    merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]

            for subOrderElement in subOrders:

                productUrl = str(subOrderElement.find("a")['href'])

                subTable = subOrderElement.find("table", {"class":"lrPad"})

                subTrs = subTable.findAll("tr")

                unitPrice=None

                offerDiscount = None

                deliveryCharges = None

                amountPaid = None

                for subTr in subTrs:

                    subTrString = str(subTr)

                    if "Unit Price" in subTrString:

                        unitPrice = re.findall(r'\d+', subTrString)[0]

                    if "Quantity" in subTrString:

                        qty = re.findall(r'\d+', subTrString)[0]

                    elif "Offer Discount" in subTrString:

                        offerDiscount =   re.findall(r'\d+', subTrString)[0]

                    elif "Delivery Charges" in subTrString:

                        deliveryCharges =   re.findall(r'\d+', subTrString)[0]

                    elif "Subtotal" in subTrString:

                        if int(qty) > 0:

                            amountPaid =   str(int(re.findall(r'\d+', subTrString)[0])/int(qty))

                        else:

                            amountPaid =   "0"

                #TODO: cashbackAmount = getCashbackAmount()

                cashbackAmount = 0

                cashbackStatus = Store.CB_PENDING

                if cashbackAmount <= 0:

                    cashbackStatus = Store.CB_NA

                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})

                if len(divs)<=0:

                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

                for div in divs:

                    productTitle = str(subOrderElement.find("a").text)

                    productUrl = "http://m.snapdeal.com/" + productUrl 

                    subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

                    subOrder.amountPaid = amountPaid

                    subOrder.deliveryCharges = deliveryCharges

                    subOrder.offerDiscount = offerDiscount

                    subOrder.unitPrice = unitPrice

                    subOrder.productCode = re.findall(r'\d+$', productUrl)[0]

                    subOrder.cashBackStatus = cashbackStatus

                    subOrder.cashBackAmount = cashbackAmount

                    trackAnchor = div.find("a")   

                    if trackAnchor is not None:

                        subOrder.tracingkUrl = str(trackAnchor['href'])

                    divStr = str(div)

                    divStr = divStr.replace("\n","").replace("\t", "")

                    for line in divStr.split("<br />"):

                        if "Suborder ID" in line:

                            subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]   

                        elif "Status" in line:

                            subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                        elif "Est. Shipping Date" in line:

                            subOrder.estimatedShippingDate = line.split(":")[1].strip()

                        elif "Est. Delivery Date" in line:

                            subOrder.estimatedDeliveryDate = line.split(":")[1].strip()

                        elif "Courier Name" in line:

                            subOrder.courierName = line.split(":")[1].strip()

                        elif "Tracking No" in line:

                            subOrder.trackingNumber = line.split(":")[1].strip()

                merchantSubOrders.append(subOrder)   

            merchantOrder.subOrders = merchantSubOrders

            #print merchantOrder

            self._saveToOrder(todict(merchantOrder))

            return True

        except:

            print "Error occurred"

            traceback.print_exc()

        return False

        #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)

        #soup.find(name, attrs, recursive, text)

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus in value:

                return key

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)

    def settlePayBack(self):

        pass

    def scrapeStoreOrders(self,):

        orders = self._getActiveOrders()

        br = getBrowserObject()

        for order in orders:

            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]

            response = br.open(url)

            page = ungzipResponse(response, br)

            #page=page.decode("utf-8")

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            sections = soup.findAll("section")

            sections.pop(0)

            sections.pop(0)

            subOrders = sections

            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

            for subOrderElement in subOrders:

                closed = True

                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})

                if len(divs)<=0:

                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

                for div in divs:

                    divStr = str(div)

                    divStr = divStr.replace("\n","").replace("\t", "")

                    updateMap = {}

                    for line in divStr.split("<br />"):

                        if "Suborder ID" in line:

                            merchantSubOrderId = re.findall(r'\d+', line)[0]   

                            findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                        elif "Status" in line:

                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                            updateMap["subOrders.$.detailedStatus"] = detailedStatus

                            status = self._getStatusFromDetailedStatus(detailedStatus) 

                            if closed:

                                closed = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                        elif "Est. Shipping Date" in line:

                            estimatedShippingDate = line.split(":")[1].strip()

                            updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate

                        elif "Est. Delivery Date" in line:

                            estimatedDeliveryDate = line.split(":")[1].strip()

                            updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate

                        elif "Courier Name" in line:

                            courierName = line.split(":")[1].strip()

                            updateMap["subOrders.$.courierName"] = courierName

                        elif "Tracking No" in line:

                            trackingNumber = line.split(":")[1].strip()

                            updateMap["subOrders.$.trackingNumber"] = trackingNumber

                    bulk.find(findMap).update({'$set' : updateMap})

                    bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed}})

            result = bulk.execute()

            pprint(result)        

    """

    This will insert records with changes only 

    """

    def _getActiveOrders(self):

        collection = self.db.merchantOrder

        collectionMap =  {"orderSuccessUrl":1, "orderId":1,"subOrders.merchantSubOrderId":1,"subOrders.closed":1}

        stores = collection.find({"closed": False, "storeId" : self.store_id}, collectionMap)

        return [store for store in stores]

    def _saveToAffiliate(self, offers):

        collection = self.db.snapdealOrderAffiliateInfo

        try:

            collection.insert(offers,continue_on_error=True)

        except pymongo.errors.DuplicateKeyError as e:

            print e.details

    def _saveToOrder(self, order):

        collection = self.db.merchantOrder

        try:

            order = collection.insert(order)

            #merchantOder 

        except Exception as e:

            traceback.print_exc()

    def _getAllOffers(self, br, token):

        allOffers = []

        nextPage = 1  

        while True:

            data = getPostData(token, nextPage)

            response = br.open(POST_URL, data)

            rmap = json.loads(ungzipResponse(response, br))

            if rmap is not None:

                rmap = rmap['response']

                if rmap is not None and len(rmap['errors'])==0:

                    allOffers += rmap['data']['data']

                    print allOffers

            nextPage += 1

            if rmap['data']['pageCount']<nextPage:

                break

        return allOffers

    def covertToObj(self,offer):

        offerData = offer['Stat']

        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 

                              offerData['datetime'], offerData['payout'], offer['Offer']['name'], offerData['ip'], offerData['conversion_sale_amount'])

        return offer1

def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):

    endDate=datetime.date.today() + datetime.timedelta(days=1)

    startDate=endDate - datetime.timedelta(days=31)

    parameters = (

        ("page",str(page)),

        ("limit",str(limit)),

        ("fields[]","Stat.offer_id"),

        ("fields[]","Stat.datetime"),

        ("fields[]","Offer.name"),

        ("fields[]","Stat.conversion_status"),

        ("fields[]","Stat.conversion_sale_amount"),

        ("fields[]","Stat.payout"),

        ("fields[]","Stat.ip"),

        ("fields[]","Stat.ad_id"),

        ("fields[]","Stat.affiliate_info1"),

        ("sort[Stat.datetime]","desc"),

        ("filters[Stat.date][conditional]","BETWEEN"),

        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),

        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),

        ("data_start",startDate.strftime('%Y-%m-%d')),

        ("data_end",endDate.strftime('%Y-%m-%d')),

        ("Method","getConversions"),

        ("NetworkId","jasper"),

        ("SessionToken",token),

    )

    #Encode the parameters

    return urllib.urlencode(parameters)

def main():

    store = getStore(3)

    store.scrapeStoreOrders()

    #store.parseOrderRawHtml(12345, "subtagId", 122323,  "html", 'https://m.snapdeal.com/purchaseMobileComplete?code=1f4166d13ea799b65aa9dea68b3e9e70&order=4509499363')

def ungzipResponse(r,b):

    headers = r.info()

    if headers['Content-Encoding']=='gzip':

        import gzip

        print "********************"

        print "Deflating gzip response"

        print "********************"

        gz = gzip.GzipFile(fileobj=r, mode='rb')

        html = gz.read()

        gz.close()

        return html

if __name__ == '__main__':

    main()

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj