Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from bson.binary import Binary

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException

from pymongo import MongoClient

import datetime

import json

import pymongo

import re

import urllib

from pprint import pprint

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

AFFILIATE_URL='http://affiliate.snapdeal.com'

POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'

ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'

CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'

class Store(main.Store):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      main.Store.ORDER_PLACED : ['In Progress','N/A'],

                      main.Store.ORDER_DELIVERED : ['Delivered'],

                      main.Store.ORDER_SHIPPED : ['In Transit'],

                      main.Store.ORDER_CANCELLED : ['Closed For Vendor Reallocation']

                      }

    def __init__(self,store_id):

        client = MongoClient('mongodb://localhost:27017/')

        self.db = client.dtr

        super(Store, self).__init__(store_id)

    def getName(self):

        return "snapdeal"

    def scrapeAffiliate(self, startDate=None, endDate=None):

        br = getBrowserObject()

        br.open(AFFILIATE_URL)

        br.select_form(nr=0)

        br.form['data[User][password]'] = PASSWORD 

        br.form['data[User][email]'] = USERNAME

        br.submit()

        response = br.open(CONFIG_URL)

        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response, br), re.IGNORECASE)[0]

        allOffers = self._getAllOffers(br, token)

        allPyOffers = [self.covertToObj(offer).__dict__ for offer in allOffers]

        self._saveToAffiliate(allPyOffers)

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        br = getBrowserObject()

        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

        response = br.open(url)

        page = ungzipResponse(response, br)

        #page=page.decode("utf-8")

        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

        #orderHead = soup.find(name, attrs, recursive, text)

        sections = soup.findAll("section")

        #print sections

        order = sections[1]

        orderTrs = order.findAll("tr")

        placedOn = str(orderTrs[0].findAll("td")[1].text)

        #Pop two section elements

        sections.pop(0) 

        sections.pop(0)

        subOrders = sections

        merchantSubOrders = []

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        merchantOrder.merchantOderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]

        for orderTr in orderTrs:

            orderTrString = str(orderTr)

            if "Total Amount" in orderTrString:

                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]

            elif "Delivery Charges" in orderTrString:

                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]

            elif "Discount Applied" in orderTrString:

                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

            elif "Paid Amount" in orderTrString:

                merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]

        for subOrderElement in subOrders:

            productUrl = str(subOrderElement.find("a")['href'])

            subTable = subOrderElement.find("table", {"class":"lrPad"})

            subTrs = subTable.findAll("tr")

            unitPrice=None

            offerDiscount = None

            deliveryCharges = None

            amountPaid = None

            for subTr in subTrs:

                subTrString = str(subTr)

                if "Unit Price" in subTrString:

                    unitPrice = re.findall(r'\d+', subTrString)[0]

                if "Quantity" in subTrString:

                    qty = re.findall(r'\d+', subTrString)[0]

                elif "Offer Discount" in subTrString:

                    offerDiscount =   re.findall(r'\d+', subTrString)[0]

                elif "Delivery Charges" in subTrString:

                    deliveryCharges =   re.findall(r'\d+', subTrString)[0]

                elif "Subtotal" in subTrString:

                    amountPaid =   str(int(re.findall(r'\d+', subTrString)[0])/int(qty))

            divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})

            if len(divs)<=0:

                raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

            for div in divs:

                productTitle = str(subOrderElement.find("a").text)

                productUrl = "http://m.snapdeal.com/" + productUrl 

                subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

                subOrder.amountPaid = amountPaid

                subOrder.deliveryCharges = deliveryCharges

                subOrder.offerDiscount = offerDiscount

                subOrder.unitPrice = unitPrice

                subOrder.productCode = re.findall(r'\d+$', productUrl)[0]

                trackAnchor = div.find("a")   

                if trackAnchor is not None:

                    subOrder.tracingkUrl = str(trackAnchor['href'])

                divStr = str(div)

                divStr = divStr.replace("\n","").replace("\t", "")

                for line in divStr.split("<br />"):

                    if "Suborder ID" in line:

                        subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]   

                    elif "Status" in line:

                        subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                    elif "Est. Shipping Date" in line:

                        subOrder.estimatedShippingDate = line.split(":")[1].strip()

                    elif "Est. Delivery Date" in line:

                        subOrder.estimatedDeliveryDate = line.split(":")[1].strip()

                    elif "Courier Name" in line:

                        subOrder.courierName = line.split(":")[1].strip()

                    elif "Tracking No" in line:

                        subOrder.trackingNumber = line.split(":")[1].strip()

            merchantSubOrders.append(subOrder)   

        merchantOrder.subOrders = merchantSubOrders

        #print merchantOrder

        self._saveToOrder(todict(merchantOrder))

        #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)

        #soup.find(name, attrs, recursive, text)

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus in value:

                return key

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)

    def scrapeStoreOrders(self,):

        orders = self._getActiveOrders()

        br = getBrowserObject()

        for order in orders:

            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]

            response = br.open(url)

            page = ungzipResponse(response, br)

            #page=page.decode("utf-8")

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            sections = soup.findAll("section")

            sections.pop(0)

            sections.pop(0)

            subOrders = sections

            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

            for subOrderElement in subOrders:

                closed = True

                divs = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})

                if len(divs)<=0:

                    raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

                for div in divs:

                    trackAnchor = div.find("a")   

                    if trackAnchor is not None:

                        tracingkUrl = str(trackAnchor['href'])

                    divStr = str(div)

                    divStr = divStr.replace("\n","").replace("\t", "")

                    updateMap = {}

                    for line in divStr.split("<br />"):

                        if "Suborder ID" in line:

                            merchantSubOrderId = re.findall(r'\d+', line)[0]   

                            findMap = {"id": order['id'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                        elif "Status" in line:

                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                            detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                            updateMap["subOrders.$.detailedStatus"] = detailedStatus

                            status = self._getStatusFromDetailedStatus(detailedStatus) 

                            if closed:

                                closed = status in [Store.ORDER_PLACED, Store.ORDER_CANCELLED]

                        elif "Est. Shipping Date" in line:

                            estimatedShippingDate = line.split(":")[1].strip()

                            updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate

                        elif "Est. Delivery Date" in line:

                            estimatedDeliveryDate = line.split(":")[1].strip()

                            updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate

                        elif "Courier Name" in line:

                            courierName = line.split(":")[1].strip()

                            updateMap["subOrders.$.courierName"] = courierName

                        elif "Tracking No" in line:

                            trackingNumber = line.split(":")[1].strip()

                            updateMap["subOrders.$.trackingNumber"] = trackingNumber

                    updateMap["subOrders.$.closed"] = closed

                    bulk.find(findMap).update({'$set' : updateMap})

            result = bulk.execute()

            pprint(result)        

    """

    This will insert records with changes only 

    """

    def _getActiveOrders(self):

        collection = self.db.merchantOrder

        stores = collection.find({"closed": False, "storeId" : self.store_id}, {"orderSuccessUrl":1, "id":1})

        return [store for store in stores]

    def _saveToAffiliate(self, offers):

        collection = self.db.snapdealOrderAffiliateInfo

        try:

            collection.insert(offers,continue_on_error=True)

        except pymongo.errors.DuplicateKeyError as e:

            print e.details

    def _saveToOrder(self, order):

        collection = self.db.merchantOrder

        try:

            order = collection.insert(order)

            #merchantOder 

        except Exception as e:

            print todict(e)

    def _getAllOffers(self, br, token):

        allOffers = []

        nextPage = 1  

        while True:

            data = getPostData(token, nextPage)

            response = br.open(POST_URL, data)

            rmap = json.loads(ungzipResponse(response, br))

            if rmap is not None:

                rmap = rmap['response']

                if rmap is not None and len(rmap['errors'])==0:

                    allOffers += rmap['data']['data']

                    print allOffers

            nextPage += 1

            if rmap['data']['pageCount']<nextPage:

                break

        return allOffers

    def covertToObj(self,offer):

        offerData = offer['Stat']

        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 

                              offerData['datetime'], offerData['payout'], offer['Offer']['name'], offerData['ip'], offerData['conversion_sale_amount'])

        return offer1

def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):

    endDate=datetime.date.today() + datetime.timedelta(days=1)

    startDate=endDate - datetime.timedelta(days=31)

    parameters = (

        ("page",str(page)),

        ("limit",str(limit)),

        ("fields[]","Stat.offer_id"),

        ("fields[]","Stat.datetime"),

        ("fields[]","Offer.name"),

        ("fields[]","Stat.conversion_status"),

        ("fields[]","Stat.conversion_sale_amount"),

        ("fields[]","Stat.payout"),

        ("fields[]","Stat.ip"),

        ("fields[]","Stat.ad_id"),

        ("fields[]","Stat.affiliate_info1"),

        ("sort[Stat.datetime]","desc"),

        ("filters[Stat.date][conditional]","BETWEEN"),

        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),

        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),

        ("data_start",startDate.strftime('%Y-%m-%d')),

        ("data_end",endDate.strftime('%Y-%m-%d')),

        ("Method","getConversions"),

        ("NetworkId","jasper"),

        ("SessionToken",token),

    )

    #Encode the parameters

    return urllib.urlencode(parameters)

def main():

    store = getStore(3)

    store.scrapeStoreOrders()

    #store.scrapeAffiliate()

    #with open ("data.txt", "r") as myfile:

    #    data=myfile.read()

    #    myfile.close()

    #store.parseOrderRawHtml(12345, "subtagId", 122323,  "html", 'https://m.snapdeal.com/purchaseMobileComplete?code=1f4166d13ea799b65aa9dea68b3e9e70&order=4509499363')

def ungzipResponse(r,b):

    headers = r.info()

    if headers['Content-Encoding']=='gzip':

        import gzip

        print "********************"

        print "Deflating gzip response"

        print "********************"

        gz = gzip.GzipFile(fileobj=r, mode='rb')

        html = gz.read()

        gz.close()

        return html

if __name__ == '__main__':

    main()

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj