Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from bs4 import BeautifulSoup

from bson.binary import Binary

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \

    Store as MStore, ungzipResponse, tprint

from dtr.storage import Mongo

from dtr.storage.Mongo import getImgSrc

from dtr.utils.utils import fetchResponseUsingProxy

from pprint import pprint

from pymongo import MongoClient

import json

import pymongo

import re

import time

import traceback

import urllib

from urlparse import urlparse, parse_qs

USERNAME='profittill2@gmail.com'

PASSWORD='spice@2020'

AFFILIATE_URL='http://affiliate.snapdeal.com'

POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'

ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'

CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'

class Store(MStore):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      MStore.ORDER_PLACED : ['in progress', 'pending for verification', 'not available', 'in process', 'processing', 'processed'],

                      MStore.ORDER_DELIVERED : ['delivered'],

                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched'],

                      MStore.ORDER_CANCELLED : ['closed for vendor reallocation', 'cancelled', 'product returned by courier', 'returned', 'n/a']

                      }

    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def getName(self):

        return "snapdeal"

    def scrapeAffiliate(self, startDate=None, endDate=None):

        br = getBrowserObject()

        br.open(AFFILIATE_URL)

        br.select_form(nr=0)

        br.form['data[User][password]'] = PASSWORD 

        br.form['data[User][email]'] = USERNAME

        br.submit()

        response = br.open(CONFIG_URL)

        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]

        print token

        allOffers = self._getAllOffers(br, token)

        self._saveToAffiliate(allOffers)

    def _setLastSaleDate(self, saleDate):

        self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})

    def _getLastSaleDate(self,):

        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})

        if lastDaySaleObj is None:

            return datetime.min

    def _parseB(self, orderId, subTagId, userId, page, orderSuccessUrl):

        soup = BeautifulSoup(page)

        orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)

        orderDetailDiv = orderDetailContainerDivs.pop(0)

        paymentDetailDiv  = orderDetailContainerDivs.pop(0)

        subOrders = orderDetailContainerDivs

        placedOn = orderDetailDiv.span.text.split(':')[1].strip()

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        merchantOrder.placedOn = placedOn

        merchantOrder.merchantOrderId = parse_qs(urlparse(orderSuccessUrl).query)['order'][0]

        paymentDivs = paymentDetailDiv.findAll('div', recursive=False)

        paymentDivs.pop(0)

        for orderTr in paymentDivs:

            orderTrString = str(orderTr)

            if "Total Amount Paid" in orderTrString:

                amountPaid = orderTr.div.find('div', {'class':'detailBlock'}).text.strip()

                merchantOrder.paidAmount = int(re.findall(r'\d+', amountPaid)[0])

            elif "Total Amount" in orderTrString:

                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]

            elif "Delivery Charges" in orderTrString:

                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]

            elif "Discount Applied" in orderTrString:

                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

            elif "Offer Discount" in orderTrString:

                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

        merchantSubOrders = []

        for subOrderElement in subOrders:

            subOrder = self.parseSubOrderB(subOrderElement, placedOn)                           

            merchantSubOrders.append(subOrder)

        merchantOrder.subOrders = merchantSubOrders

        return merchantOrder

    def _parse(self, orderId, subTagId, userId, page, orderSuccessUrl):

        #page=page.decode("utf-8")

        soup = BeautifulSoup(page)

        #orderHead = soup.find(name, attrs, recursive, text)

        sections = soup.findAll("section")

        #print sections

        order = sections[1]

        orderTrs = order.findAll("tr")

        placedOn = str(orderTrs[0].findAll("td")[1].text)

        #Pop two section elements

        sections.pop(0) 

        sections.pop(0)

        subOrders = sections

        merchantSubOrders = []

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        merchantOrder.placedOn = placedOn

        merchantOrder.merchantOrderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]

        for orderTr in orderTrs:

            orderTrString = str(orderTr)

            if "Total Amount" in orderTrString:

                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]

            elif "Delivery Charges" in orderTrString:

                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]

            elif "Discount Applied" in orderTrString:

                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

            elif "Paid Amount" in orderTrString:

                merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]

        for subOrderElement in subOrders:

            subOrders = self.parseSubOrder(subOrderElement, placedOn)                           

            merchantSubOrders.extend(subOrders)   

        merchantOrder.subOrders = merchantSubOrders

        return merchantOrder

    def parseSubOrder(self, subOrderElement, placedOn):

        subOrders = []

        productUrl = str(subOrderElement.find("a")['href'])

        subTable = subOrderElement.find("table", {"class":"lrPad"})

        subTrs = subTable.findAll("tr")

        unitPrice=None

        offerDiscount = 0

        deliveryCharges = None

        amountPaid = None

        amount = 0

        sdCash = 0

        unitPrice = 0

        for subTr in subTrs:

            subTrString = str(subTr)

            if "Unit Price" in subTrString:

                unitPrice = int(re.findall(r'\d+', subTrString)[0])

            if "Quantity" in subTrString:

                qty = int(re.findall(r'\d+', subTrString)[0])

            elif "Offer Discount" in subTrString:

                offerDiscount +=   int(re.findall(r'\d+', subTrString)[0])

            elif "SD Cash" in subTrString:

                sdCash =   int(re.findall(r'\d+', subTrString)[0])

            elif "Delivery Charges" in subTrString:

                deliveryCharges =   int(re.findall(r'\d+', subTrString)[0])

            elif "Subtotal" in subTrString:

                if int(qty) > 0:

                    amountPaid =   int(re.findall(r'\d+', subTrString)[0])/qty

                else:

                    amountPaid =   0

        if qty>0:

            amount = unitPrice - offerDiscount - sdCash

            amount = 0 if amount < 0 else amount

        div1 = subOrderElement.find("div", {"class": "blk lrPad subordrs"})

        if div1 is None:

            raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

        for strDiv in str(div1).split("<div class=\"seperator\"></div>"):

            div = BeautifulSoup(strDiv)

            productTitle = str(subOrderElement.find("a").text)

            productUrl = "http://m.snapdeal.com/" + productUrl 

            subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

            subOrder.amountPaid = amountPaid

            subOrder.deliveryCharges = deliveryCharges

            subOrder.offerDiscount = offerDiscount

            subOrder.unitPrice = int(unitPrice)

            subOrder.productCode = re.findall(r'\d+$', productUrl)[0]

            subOrder.imgUrl = Mongo.getImgSrc(subOrder.productCode, self.store_id).get('thumbnail')

            cashbackStatus = Store.CB_NA

            cashbackAmount = 0

            percentage = 0

            if amount > 0:

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)

                if cashbackAmount > 0:

                    cashbackStatus = Store.CB_PENDING

            subOrder.cashBackStatus = cashbackStatus

            subOrder.cashBackAmount = cashbackAmount

            subOrder.cashBackPercentage = percentage

            trackAnchor = div.find("a")   

            if trackAnchor is not None:

                subOrder.tracingkUrl = str(trackAnchor['href'])

            divStr = str(div)

            divStr = divStr.replace("\n","").replace("\t", "")

            for line in divStr.split("<br />"):

                if "Suborder ID" in line:

                    subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]   

                elif "Status" in line:

                    print line

                    subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                elif "Est. Shipping Date" in line:

                    subOrder.estimatedShippingDate = line.split(":")[1].strip()

                elif "Est. Delivery Date" in line:

                    subOrder.estimatedDeliveryDate = line.split(":")[1].strip()

                elif "Courier Name" in line:

                    subOrder.courierName = line.split(":")[1].strip()

                elif "Tracking No" in line:

                    subOrder.trackingNumber = line.split(":")[1].strip()

            subOrders.append(subOrder)

        return subOrders

    def parseSubOrderB(self, subOrderElement, placedOn):

        subOrders = []

        prodDivs = subOrderElement.findAll('div', recursive=False)

        prodDetailDiv = prodDivs[1].findAll('div', recursive=False)

        offerDiscount = 0

        deliveryCharges = None

        amountPaid = 0

        sdCash = 0

        unitPrice = 0

        paymentDivs = prodDivs[2].findAll('div', recursive=False)

        for paymentDiv in paymentDivs:

            strPaymentDiv = str(paymentDiv)

            if "Unit Price" in strPaymentDiv:

                unitPrice = int(re.findall(r'\d+', strPaymentDiv)[0])

            if "Offer Discount" in strPaymentDiv:

                offerDiscount = int(re.findall(r'\d+', strPaymentDiv)[0])

            elif "SD Cash" in strPaymentDiv:

                sdCash =   int(re.findall(r'\d+', strPaymentDiv)[0])

            elif "Delivery Charges" in strPaymentDiv:

                deliveryCharges =   int(re.findall(r'\d+', strPaymentDiv)[0])

            elif "Subtotal" in strPaymentDiv:

                amountPaid =   int(re.findall(r'\d+', paymentDiv.find('div', {'class':'itemPriceDetail'}).text)[0])

        amount = unitPrice - offerDiscount - sdCash

        imgDiv = prodDetailDiv[0]

        otherDiv = prodDetailDiv[1]

        productTitle = otherDiv.find('div',{'class':'orderName'}).text.strip()

        productUrl = imgDiv.a['href']

        subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

        subOrder.merchantSubOrderId = prodDivs[0].text.split(':')[1].strip()

        subOrder.detailedStatus = otherDiv.find('div',{'class':'orderStatus'}).span.text.strip()

        deliveryStatus = otherDiv.find('div',{'class':'orderDelivery'})

        if deliveryStatus is not None:

            delString = deliveryStatus.text.strip()

            arr = delString.split(':')

            if "On" in arr[0]:

                subOrder.deliveredOn = arr[1].strip()

            elif "Exp. Delivery by" in arr[0]:

                subOrder.estimatedDeliveryDate = arr[1].strip()

            else:

                subOrder.estimatedShippingDate = arr[1].strip()

        subOrder.imgUrl = imgDiv.a.img['src']

        subOrder.productCode = re.findall(r'\d+$', productUrl)[0]

        subOrder.deliveryCharges = deliveryCharges

        subOrder.offerDiscount = offerDiscount

        subOrder.unitPrice = int(unitPrice)

        cashbackStatus = Store.CB_NA

        cashbackAmount = 0

        percentage = 0

        if amountPaid > 0:

            (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)

            if cashbackAmount > 0:

                cashbackStatus = Store.CB_PENDING

        subOrder.cashBackStatus = cashbackStatus

        subOrder.cashBackAmount = cashbackAmount

        subOrder.cashBackPercentage = percentage

        courierDet = subOrderElement.find('div', {'class':'courierDetail'})

        if courierDet is not None:

            subOrder.courierName = courierDet.span.text.strip()

        trackingDet = subOrderElement.find('div', {'class':'trackingNo'})

        if trackingDet is not None:

            subOrder.trackingUrl = trackingDet.span.a['href']

            subOrder.trackingNumber = trackingDet.span.a.text.strip()

        subOrders.append(subOrder)

        return subOrder

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

                    #print merchantOrder

        resp = {}

        try:

            br = getBrowserObject()

            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

            page = br.open(url)

            page = ungzipResponse(page)

            try:

                merchantOrder = self._parseB(orderId, subTagId, userId, page, orderSuccessUrl)

            except:

                traceback.print_exc()

                merchantOrder = self._parse(orderId, subTagId, userId, page, orderSuccessUrl)

            merchantOrder.orderTrackingUrl = url

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp

        except:

            print "Error occurred"

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            return resp

        #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)

        #soup.find(name, attrs, recursive, text)

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus, self.store_id

        return None

    def scrapeStoreOrders(self,):

        #collectionMap = {'palcedOn':1}

        orders = self._getActiveOrders()

        for order in orders:

            print "Order", self.store_name, order['orderId']

            try:

                url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]

                page = fetchResponseUsingProxy(url)

                #page=page.decode("utf-8")

                soup = BeautifulSoup(page)

                try:

                    self.tryBParsing(order, soup)

                except:

                    traceback.print_exc()

                    sections = soup.findAll("section")

                    orderEl = sections[1]

                    orderTrs = orderEl.findAll("tr")

                    placedOn = str(orderTrs[0].findAll("td")[1].text)

                    sections.pop(0)

                    sections.pop(0)

                    subOrders = sections

                    bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                    closed = True

                    for subOrderElement in subOrders:

                        div1 = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})

                        if len(div1)<=0:

                            raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

                        subOrder = None

                        breakFlag = False

                        for strDiv in str(div1).split("<div class=\"seperator\"></div>"):

                            div = BeautifulSoup(strDiv)

                            divStr = str(div)

                            divStr = divStr.replace("\n","").replace("\t", "")

                            updateMap = {}

                            for line in divStr.split("<br />"):

                                if "Suborder ID" in line:

                                    merchantSubOrderId = re.findall(r'\d+', line)[0]

                                    #break if suborder is inactive   

                                    subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

                                    if subOrder is None:

                                        subOrders = self.parseSubOrder(subOrderElement, placedOn)

                                        self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict(subOrders)}}})

                                        print "Added new suborders to Order id - ", order['orderId']

                                        closed = False

                                        breakFlag = True

                                        break

                                    elif subOrder['closed']:

                                        breakFlag = True

                                        break

                                    else: 

                                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                                elif "Status :" in line:

                                    detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                                    updateMap["subOrders.$.detailedStatus"] = detailedStatus

                                    status = self._getStatusFromDetailedStatus(detailedStatus) 

                                    closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                                    if status is not None:

                                        updateMap["subOrders.$.status"] = status

                                    if detailedStatus == 'Closed For Vendor Reallocation':

                                        #if it is more than 6hours mark closed.

                                        closeAt = subOrder.get("closeAt") 

                                        if closeAt is None:

                                            closeAt = datetime.now() + timedelta(hours=6)

                                            updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")

                                        else:

                                            closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")

                                            if datetime.now() > closeAt:

                                                closedStatus = True

                                    if closedStatus:

                                        #if status is closed then change the paybackStatus accordingly

                                        updateMap["subOrders.$.closed"] = True

                                        if status == Store.ORDER_DELIVERED:

                                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                                        elif status == Store.ORDER_CANCELLED:

                                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                                    else:

                                        closed = False

                                elif "Est. Shipping Date" in line:

                                    estimatedShippingDate = line.split(":")[1].strip()

                                    updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate

                                elif "Est. Delivery Date" in line:

                                    estimatedDeliveryDate = line.split(":")[1].strip()

                                    updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate

                                elif "Courier Name" in line:

                                    courierName = line.split(":")[1].strip()

                                    updateMap["subOrders.$.courierName"] = courierName

                                elif "Tracking No" in line:

                                    trackingNumber = line.split(":")[1].strip()

                                    updateMap["subOrders.$.trackingNumber"] = trackingNumber

                            if breakFlag:

                                continue

                            bulk.find(findMap).update({'$set' : updateMap})

                        bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})

                    result = bulk.execute()

                    tprint(result)

            except:

                tprint("Could not update " + str(order['orderId']) + "For store " + self.getName())

                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})

                traceback.print_exc()                

    def tryBParsing(self, order, soup):

        orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)

        orderDetailDiv = orderDetailContainerDivs.pop(0)

        placedOn = orderDetailDiv.span.text.split(':')[1].strip()

        orderDetailContainerDivs.pop(0)

        subOrders = orderDetailContainerDivs

        bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

        closed = True

        for subOrderElement in subOrders:

            prodDivs = subOrderElement.findAll('div', recursive=False)

            merchantSubOrderId = prodDivs[0].text.split(':')[1].strip()

            prodDetailDiv = prodDivs[1].findAll('div', recursive=False)

            imgDiv = prodDetailDiv[0]

            otherDiv = prodDetailDiv[1]

            subOrder = None

            breakFlag = False

            subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

            if subOrder is None:

                subOrders = self.parseSubOrderB(subOrderElement, placedOn)

                self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrders])}}})

                print "Added new suborders to Order id - ", order['orderId']

                closed = False

                continue

            elif subOrder['closed']:

                continue

            else: 

                findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                updateMap = {}

                prodDivs = subOrderElement.findAll('div', recursive=False)

                prodDetailDiv = prodDivs[1].findAll('div', recursive=False)

                otherDiv = prodDetailDiv[1]

                detailedStatus = otherDiv.find('div',{'class':'orderStatus'}).span.text.strip()

                updateMap["subOrders.$.detailedStatus"] = detailedStatus

                status = self._getStatusFromDetailedStatus(detailedStatus) 

                closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                if status is not None:

                    updateMap["subOrders.$.status"] = status

                if detailedStatus == 'Closed For Vendor Reallocation':

                    #if it is more than 6hours mark closed.

                    closeAt = subOrder.get("closeAt") 

                    if closeAt is None:

                        closeAt = datetime.now() + timedelta(hours=6)

                        updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")

                    else:

                        closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")

                        if datetime.now() > closeAt:

                            closedStatus = True

                if closedStatus:

                    #if status is closed then change the paybackStatus accordingly

                    updateMap["subOrders.$.closed"] = True

                    if status == Store.ORDER_DELIVERED:

                        if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                            updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                    elif status == Store.ORDER_CANCELLED:

                        if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                            updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                else:

                    closed = False

                deliveryStatus = otherDiv.find('div',{'class':'orderDelivery'})

                if deliveryStatus is not None:

                    delString = deliveryStatus.text.strip()

                    arr = delString.split(':')    

                    if "On" in arr[0]:

                        updateMap['subOrders.$.deliveredOn'] = arr[1].strip()

                    elif "Exp. Delivery by" in arr[0]:

                        updateMap['subOrders.$.estimatedDeliveryDate'] = arr[1].strip()

                    else:

                        updateMap['subOrders.$.estimatedShippingDate'] = arr[1].strip()

                courierDet = subOrderElement.find('div', {'class':'courierDetail'})

                if courierDet is not None:

                    updateMap['subOrders.$.courierName'] = courierDet.span.text.strip()

                trackingDet = subOrderElement.find('div', {'class':'trackingNo'})

                if trackingDet is not None:

                    updateMap['subOrders.$.trackingUrl'] = trackingDet.span.a['href']

                    updateMap['subOrders.$.trackingNumber'] = trackingDet.span.a.text.strip()                    

            bulk.find(findMap).update({'$set' : updateMap})

        bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})

        result = bulk.execute()

        tprint(result)

    def _saveToAffiliate(self, offers):

        collection = self.db.snapdealOrderAffiliateInfo

        mcollection = self.db.merchantOrder

        for offer in offers:

            offer = self.covertToObj(offer)

            collection.update({"adId":offer.adId, "saleAmount":offer.saleAmount, "payOut":offer.payOut},{"$set":todict(offer)}, upsert=True)

            mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True}, {"$set":{"subOrders.$.missingAff":False}})

    def _getAllOffers(self, br, token):

        allOffers = []

        nextPage = 1  

        while True:

            data = getPostData(token, nextPage)

            response = br.open(POST_URL, data)

            rmap = json.loads(ungzipResponse(response))

            if rmap is not None:

                rmap = rmap['response']

                print rmap

                if rmap is not None and len(rmap['errors'])==0:

                    allOffers += rmap['data']['data']

            nextPage += 1

            if rmap['data']['pageCount']<nextPage:

                break

        return allOffers

    def covertToObj(self,offer):

        offerData = offer['Stat']

        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 

                              offerData['datetime'], int(float(offerData['payout'])), offer['Offer']['name'], offerData['ip'], int(float(offerData['conversion_sale_amount'])))

        offer1.saleTime = int(time.mktime(datetime.strptime(offer1.saleDate, "%Y-%m-%d %H:%M:%S").timetuple()))

        return offer1

def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):

    endDate=date.today() + timedelta(days=1)

    startDate=endDate - timedelta(days=31)

    parameters = (

        ("page",str(page)),

        ("limit",str(limit)),

        ("fields[]","Stat.offer_id"),

        ("fields[]","Stat.datetime"),

        ("fields[]","Offer.name"),

        ("fields[]","Stat.conversion_status"),

        ("fields[]","Stat.conversion_sale_amount"),

        ("fields[]","Stat.payout"),

        ("fields[]","Stat.ip"),

        ("fields[]","Stat.ad_id"),

        ("fields[]","Stat.affiliate_info1"),

        ("sort[Stat.datetime]","desc"),

        ("filters[Stat.date][conditional]","BETWEEN"),

        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),

        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),

        ("data_start",startDate.strftime('%Y-%m-%d')),

        ("data_end",endDate.strftime('%Y-%m-%d')),

        ("Method","getConversions"),

        ("NetworkId","jasper"),

        ("SessionToken",token),

    )

    #Encode the parameters

    return urllib.urlencode(parameters)

def main():

    #print todict([1,2,"3"])

    store = getStore(3)

    #store.parseOrderRawHtml(33222, "32323", 2, "323243", "https://m.snapdeal.com/purchaseMobileComplete?code=1131a4add85aefb40a924d71cbf921bd&order=5822561503")

    store.scrapeStoreOrders()

    #store._isSubOrderActive(8, "5970688907")

    #store.scrapeAffiliate()

    #store.scrapeStoreOrders()

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj

if __name__ == '__main__':

    main()