Subversion Repositories SmartDukaan

'''
Created on Jan 15, 2015

@author: Manish 
'''
from bs4 import BeautifulSoup
from bson.binary import Binary
from datetime import datetime, date, timedelta
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
    Store as MStore, ungzipResponse, tprint
from dtr.storage import Mongo
from dtr.storage.Mongo import getImgSrc
from dtr.utils.utils import fetchResponseUsingProxy
from pprint import pprint
from pymongo import MongoClient
import json
import pymongo
import re
import time
import traceback
import urllib
from urlparse import urlparse, parse_qs

USERNAME='profittill2@gmail.com'
PASSWORD='spice@2020'
AFFILIATE_URL='http://affiliate.snapdeal.com'
POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
ORDER_TRACK_URL='http://www.shopclues.com/index.php?dispatch=order_lookup.details'
CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'
BASE_URL= 'http://www.shopclues.com'
BASE_MURL= 'http://m.shopclues.com'

class Store(MStore):
    
    '''
    This is to map order statuses of our system to order statuses of snapdeal.
    And our statuses will change accordingly.
    
    '''
    OrderStatusMap = {
                      MStore.ORDER_PLACED : ['payment successful', 'new order - cod confirmation pending', 'processing', 'quality check','on schedule', 'processing - pickup-initiated', 'processing - ready to dispatch'],
                      MStore.ORDER_DELIVERED : ['delivered', 'complete'],
                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched','shipped','order handed to courier'],
                      MStore.ORDER_CANCELLED : ['payment failed', 'canceled', 'payment declined', 'order on hold - cancellation requested by customer', 'courier returned']
                      }
    OrderStatusConfirmationMap= {
                                 "Payment Successful" : "P",
                                 "Order Declined" : "D",
                                 "New Order - COD confirmation Pending" : "O"
                                 }
    
    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE
    '''
    def scrapeAffiliate(self, startDate=None, endDate=None):
        br = getBrowserObject()
        br.open(AFFILIATE_URL)
        br.select_form(nr=0)
        br.form['data[User][password]'] = PASSWORD 
        br.form['data[User][email]'] = USERNAME
        br.submit()
        response = br.open(CONFIG_URL)
        
        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
        print token
        allOffers = self._getAllOffers(br, token)
        self._saveToAffiliate(allOffers)
    '''
    
    def _setLastSaleDate(self, saleDate):
        self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})
    
    def getName(self):
        return "shopclues"    
        
    def _getLastSaleDate(self,):
        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})
        if lastDaySaleObj is None:
            return datetime.min
        
    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, value in Store.OrderStatusMap.iteritems():
            if detailedStatus.lower() in value:
                return key
        print "Detailed Status need to be mapped", detailedStatus, self.store_id
        return None
        
    def _getSingleSubOrderMap(self, orderId, orderStatus, statusTime, soup, subTagId):
        productDetailsMap = {}
        orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
        orderTable.pop(0)
        productDetailsSubMap = {}
        for orderTr in orderTable:
            cols = orderTr.find_all('td')
            product_details = cols[0].find_all('a')
            #print product_details
            productUrl = product_details[0].get('href')
            productName = product_details[0].contents[0].strip()
            quantity = int(cols[1].text.strip())
            sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
            discount = long(cols[3].text.strip().replace("Rs.",""))
            subtotal = long(cols[4].text.strip().replace("Rs.",""))
            productDetailsSubMap['productUrl']=BASE_MURL+productUrl
            productDetailsSubMap['productName']=productName
            productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL+'&order_id=' +str(orderId)+'&email_id='+ subTagId.split('$')[1]
            productDetailsSubMap['sellingPrice']=sellingPrice
            productDetailsSubMap['quantity']=quantity
            productDetailsSubMap['discount']=discount
            productDetailsSubMap['subtotal']=subtotal
            productDetailsSubMap['parentOrderId']=orderId
            br = getBrowserObject()
            productPage = br.open(BASE_MURL+productUrl)
            productPageHeaders = str(productPage.info()).split('\n')
            productPage = ungzipResponse(productPage)
            jsonProductResponse = None
            for header in productPageHeaders:
                header = header.split(':')
                if header[0] == 'Content-Type' and 'json' in header[1]:
                    jsonProductResponse = json.loads(productPage)
            productPageSoup= None
            if jsonProductResponse is not None:
                productPageSoup = BeautifulSoup(jsonProductResponse['text'])
            else:
                productPageSoup = BeautifulSoup(productPage)
            productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
            lengthProductCodeArr= len(productCodeArray)
            productCode = productCodeArray[lengthProductCodeArr-2]
            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
            productImgUrl = ''
            if allproductImageTags is not None and len(allproductImageTags)>0:
                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
            productDetailsSubMap['productCode']=productCode
            productDetailsSubMap['imgUrl']=productImgUrl
            
        productDetailsSubMap['subOrderStatus']=orderStatus
        productDetailsSubMap['subOrderStatusTime']=statusTime
        productDetailsMap[orderId]=productDetailsSubMap
        return productDetailsMap
    
    def _getMultiSubOrdersMap(self, orderId, soup):
        orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
        orderTable.pop(0)
        productDetailsMap = {}
        for orderTr in orderTable:
            productDetailsSubMap = {}
            cols = orderTr.find_all('td')
            product_details = cols[0].find_all('a')
            #print product_details
            subOrderId= product_details[1].contents[0].strip()
            productUrl = product_details[0].get('href')
            productName = product_details[0].contents[0].strip()
            subOrderTrackingUrl = product_details[1].get('href')
            quantity = int(cols[1].text.strip())
            sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
            discount = long(cols[3].text.strip().replace("Rs.",""))
            subtotal = long(cols[4].text.strip().replace("Rs.",""))
            productDetailsSubMap['productUrl']=BASE_MURL+productUrl
            productDetailsSubMap['productName']=productName
            productDetailsSubMap['subOrderTrackingUrl']=BASE_URL+subOrderTrackingUrl
            productDetailsSubMap['sellingPrice']=sellingPrice
            productDetailsSubMap['quantity']=quantity
            productDetailsSubMap['discount']=discount
            productDetailsSubMap['subtotal']=subtotal
            br = getBrowserObject()
            productPage = br.open(BASE_MURL+productUrl)
            productPageHeaders = str(productPage.info()).split('\n')
            productPage = ungzipResponse(productPage)
            jsonProductResponse = None
            for header in productPageHeaders:
                header = header.split(':')
                if header[0] == 'Content-Type' and 'json' in header[1]:
                    jsonProductResponse = json.loads(productPage)
            productPageSoup= None
            if jsonProductResponse is not None:
                productPageSoup = BeautifulSoup(jsonProductResponse['text'])
            else:
                productPageSoup = BeautifulSoup(productPage)
            productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
            lengthProductCodeArr= len(productCodeArray)
            productCode = productCodeArray[lengthProductCodeArr-2]
            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
            productImgUrl = ''
            if allproductImageTags is not None and len(allproductImageTags)>0:
                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
            productDetailsSubMap['productCode']=productCode
            productDetailsSubMap['imgUrl']=productImgUrl
            br1 = getBrowserObject()
            orderTrackingPage = br1.open(BASE_URL+subOrderTrackingUrl)
            headers = str(orderTrackingPage.info()).split('\n')
            orderTrackingPage= ungzipResponse(orderTrackingPage)
            jsonResponse = None
            for header in headers:
                header = header.split(':')
                if header[0] == 'Content-Type' and 'json' in header[1]:
                    jsonResponse = json.loads(orderTrackingPage)
            
            orderTrackingPageSoup = None
            if jsonResponse is not None:
                orderTrackingPageSoup = BeautifulSoup(str(jsonResponse['text']))
            else:
                orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)
            subOrderStatusList = orderTrackingPageSoup.findAll(attrs={'class' : 'price ord_status'})
            subOrderStatus = subOrderStatusList[0].contents[0].strip()
            subOrderStatusTime= orderTrackingPageSoup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
            productDetailsSubMap['subOrderStatus'] = subOrderStatus
            productDetailsSubMap['subOrderStatusTime'] = subOrderStatusTime
            productDetailsSubMap['parentOrderId']=orderId
            
            productDetailsMap[subOrderId]=productDetailsSubMap
            
        return productDetailsMap
    
    def updateCashbackInSubOrders(self, subOrders):
        for subOrder in subOrders:
            cashbackStatus = Store.CB_NA
            cashbackAmount = 0
            percentage = 0
            amount = subOrder.amountPaid
            if amount > 0:
                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)
                if cashbackAmount > 0:
                    cashbackStatus = Store.CB_PENDING
            subOrder.cashBackStatus = cashbackStatus
            subOrder.cashBackAmount = cashbackAmount
            subOrder.cashBackPercentage = percentage
        return subOrders
        
    def _parseOrders(self, orderId, subTagId, userId, page, orderSuccessUrl):
        soup = BeautifulSoup(page)
        productDetailsMap = {}
        orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})
        paymentFields = soup.findAll(attrs={'class' : 'box_paymentcalculations_row'})
        orderDateList = soup.findAll(attrs={'class':'price ord_date'})
        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("  ","")
        ordersubtotal=0
        ordercluebucks=0
        ordershippingcost=0
        ordertotal=0
        
        for val in paymentFields:
            for value in val.contents:
                if value is not None:
                    if 'div' in str(value).strip():
                        if 'Subtotal' in value.text.strip():
                            ordersubtotal = long(val.contents[3].text.strip().replace("Rs.",""))
                            print ordersubtotal
                        if 'Shipping Cost' in value.text.strip():
                            ordershippingcost = long(val.contents[3].text.strip().replace("Rs.",""))
                            print ordershippingcost
                        if 'Clue' in value.text.strip():
                            ordercluebucks = long(val.contents[3].text.strip().replace("Rs.",""))
                            print ordercluebucks
                        if 'Total' in value.text.strip():
                            ordertotal = long(val.contents[3].text.strip().replace("Rs.",""))
                            print ordertotal
                            
        if orderStatusList is not None and len(orderStatusList)>0:
            orderStatus = orderStatusList[0].contents[0].strip()
            statusTime= soup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
            productDetailsMap = self._getSingleSubOrderMap(orderId, orderStatus, statusTime, soup, subTagId)
        else:
            productDetailsMap = self._getMultiSubOrdersMap(orderId, soup)
            
        merchantOrder = Order(orderId, userId, subTagId.split('$')[0], self.store_id, orderSuccessUrl)
        merchantOrder.placedOn = placedOn
        merchantOrder.merchantOrderId = orderId
        merchantOrder.paidAmount = ordertotal
        totalOrdersAmount = 0
        totalDiscount = 0
        subOrders= []
        for key in productDetailsMap:
            subOrderDetail = productDetailsMap.get(key)
            totalOrdersAmount = totalOrdersAmount  + (subOrderDetail['sellingPrice'] * subOrderDetail['quantity'])
            totalDiscount = totalDiscount + (subOrderDetail['discount'] * subOrderDetail['quantity'])
            subOrder = SubOrder(subOrderDetail['productName'], subOrderDetail['productUrl'], placedOn, subOrderDetail['subtotal'])
            subOrder.merchantSubOrderId = key
            subOrder.detailedStatus = subOrderDetail['subOrderStatus']
            subOrder.imgUrl = subOrderDetail['imgUrl']
            subOrder.offerDiscount = subOrderDetail['discount'] * subOrderDetail['quantity']
            subOrder.unitPrice = subOrderDetail['sellingPrice']
            subOrder.productCode = subOrderDetail['productCode']
            subOrder.amountPaid = subOrderDetail['subtotal']
            subOrder.quantity = subOrderDetail['quantity']
            subOrders.append(subOrder)
        print totalOrdersAmount, totalDiscount
        merchantOrder.totalAmount = totalOrdersAmount
        merchantOrder.discountApplied = totalDiscount
        merchantOrder.deliveryCharges = ordershippingcost
        merchantOrder.subOrders = self.updateCashbackInSubOrders(subOrders)
            
        return merchantOrder
        
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
                    #print merchantOrder
        resp = {}
        try:
            br = getBrowserObject()
            url = ORDER_TRACK_URL +'&order_id=' +str(orderId)+'&email_id='+ subTagId.split('$')[1]
            page = br.open(url)
            headers = str(page.info()).split('\n')
            page = ungzipResponse(page)
            jsonResponse = None
            for header in headers:
                header = header.split(':')
                if header[0] == 'Content-Type' and 'json' in header[1]:
                    jsonResponse = json.loads(page)
            if jsonResponse is not None:
                page = jsonResponse['text']
            
            merchantOrder = self._parseOrders(orderId, subTagId, userId, page, orderSuccessUrl)
            
            merchantOrder.orderTrackingUrl = url
               
            if self._saveToOrder(todict(merchantOrder)):
                resp['result'] = 'ORDER_CREATED'
            else:
                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                
            return resp
        except:
            print "Error occurred"
            traceback.print_exc()
            resp['result'] = 'ORDER_NOT_CREATED'
    
    def parseSingleSubOrder(self, soup, emailId, subOrderId, subOrderStatus):
        orderDateList = soup.findAll(attrs={'class':'price ord_date'})
        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("  ","")    
        orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
        orderTable.pop(0) 
        subOrders =[]   
        for orderTr in orderTable:
            cols = orderTr.find_all('td')
            product_details = cols[0].find_all('a')
            #print product_details
            productUrl = product_details[0].get('href')
            productName = product_details[0].contents[0].strip()
            quantity = int(cols[1].text.strip())
            sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
            discount = long(cols[3].text.strip().replace("Rs.",""))
            subtotal = long(cols[4].text.strip().replace("Rs.",""))
            br = getBrowserObject()
            productPage = br.open(BASE_MURL+productUrl)
            productPageHeaders = str(productPage.info()).split('\n')
            productPage = ungzipResponse(productPage)
            jsonProductResponse = None
            for header in productPageHeaders:
                header = header.split(':')
                if header[0] == 'Content-Type' and 'json' in header[1]:
                    jsonProductResponse = json.loads(productPage)
            productPageSoup= None
            if jsonProductResponse is not None:
                productPageSoup = BeautifulSoup(jsonProductResponse['text'])
            else:
                productPageSoup = BeautifulSoup(productPage)
            productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
            lengthProductCodeArr= len(productCodeArray)
            productCode = productCodeArray[lengthProductCodeArr-2]
            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
            productImgUrl = ''
            if allproductImageTags is not None and len(allproductImageTags)>0:
                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
                
            subOrder = SubOrder(productName, productUrl, placedOn, subtotal)
            subOrder.merchantSubOrderId = subOrderId
            subOrder.detailedStatus = subOrderStatus
            subOrder.imgUrl = productImgUrl
            subOrder.offerDiscount = discount*quantity
            subOrder.unitPrice = sellingPrice
            subOrder.productCode = productCode
            subOrder.amountPaid = subtotal
            subOrder.quantity = quantity
            
            subOrders.append(subOrder)
            
        subOrders = self.updateCashbackInSubOrders(subOrders)
        return subOrders[0]
    
    def parseMultiSubOrders(self, soup):
        orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)
        orderTable.pop(0)
        orderDateList = soup.findAll(attrs={'class':'price ord_date'})
        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("  ","")    
        subOrders = []
        for orderTr in orderTable:
            productDetailsSubMap = {}
            cols = orderTr.find_all('td')
            product_details = cols[0].find_all('a')
            #print product_details
            subOrderId= product_details[1].contents[0].strip()
            productUrl = product_details[0].get('href')
            productName = product_details[0].contents[0].strip()
            subOrderTrackingUrl = product_details[1].get('href')
            quantity = int(cols[1].text.strip())
            sellingPrice = long(cols[2].text.strip().replace("Rs.",""))
            discount = long(cols[3].text.strip().replace("Rs.",""))
            subtotal = long(cols[4].text.strip().replace("Rs.",""))
            br = getBrowserObject()
            productPage = br.open(BASE_MURL+productUrl)
            productPageHeaders = str(productPage.info()).split('\n')
            productPage = ungzipResponse(productPage)
            jsonProductResponse = None
            for header in productPageHeaders:
                header = header.split(':')
                if header[0] == 'Content-Type' and 'json' in header[1]:
                    jsonProductResponse = json.loads(productPage)
            productPageSoup= None
            if jsonProductResponse is not None:
                productPageSoup = BeautifulSoup(jsonProductResponse['text'])
            else:
                productPageSoup = BeautifulSoup(productPage)
            productCodeArray = str(productPageSoup.find("form", {'class':'buy-form'}).findAll('input', recursive=False)[0]).split('"')
            lengthProductCodeArr= len(productCodeArray)
            productCode = productCodeArray[lengthProductCodeArr-2]
            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})
            productImgUrl = ''
            if allproductImageTags is not None and len(allproductImageTags)>0:
                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()
            productDetailsSubMap['productCode']=productCode
            productDetailsSubMap['imgUrl']=productImgUrl
            br1 = getBrowserObject()
            orderTrackingPage = br1.open(BASE_URL+subOrderTrackingUrl)
            headers = str(orderTrackingPage.info()).split('\n')
            orderTrackingPage= ungzipResponse(orderTrackingPage)
            jsonResponse = None
            for header in headers:
                header = header.split(':')
                if header[0] == 'Content-Type' and 'json' in header[1]:
                    jsonResponse = json.loads(orderTrackingPage)
            
            orderTrackingPageSoup = None
            if jsonResponse is not None:
                orderTrackingPageSoup = BeautifulSoup(str(jsonResponse['text']))
            else:
                orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)
            subOrderStatusList = orderTrackingPageSoup.findAll(attrs={'class' : 'price ord_status'})
            subOrderStatus = subOrderStatusList[0].contents[0].strip()
            subOrderStatusTime= orderTrackingPageSoup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()
            
            subOrder = SubOrder(productName, productUrl, placedOn, subtotal)
            subOrder.merchantSubOrderId = subOrderId
            subOrder.detailedStatus = subOrderStatus
            subOrder.imgUrl = productImgUrl
            subOrder.offerDiscount = discount*quantity
            subOrder.unitPrice = sellingPrice
            subOrder.productCode = productCode
            subOrder.amountPaid = subtotal
            subOrder.quantity = quantity
            
            subOrders.append(subOrder)
            
            return self.updateCashbackInSubOrders(subOrders)
            
    def scrapeStoreOrders(self,):
        #collectionMap = {'palcedOn':1}
        searchMap = {}
        collectionMap = {"orderTrackingUrl":1}
        orders = self._getActiveOrders(searchMap,collectionMap)
        for order in orders:
            print "Order", self.store_name, order['orderId'], order['orderTrackingUrl']
            url = order['orderTrackingUrl']
            page = fetchResponseUsingProxy(url)
            soup = BeautifulSoup(page)
            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
            closed = True
            orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})
            if orderStatusList is not None and len(orderStatusList)>0:
                subOrderId = soup.findAll(attrs={'class':'price ord_no'})[0].text.strip()
                subOrder =  self._isSubOrderActive(order, subOrderId)
                orderStatus = orderStatusList[0].contents[0].strip()
                if subOrder is None:
                    try:
                        subOrder = self.parseSingleSubOrder(soup, order['orderTrackingUrl'].split('email_id=')[1], subOrderId, orderStatus)
                        if subOrder is None:
                            continue
                        self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrder])}}})
                        print "Added new suborders to Order id - ", order['orderId']
                        closed = False
                    except:
                        pass
                    continue
                elif subOrder['closed']:
                    continue
                else:
                    findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}
                    updateMap = {}
                    updateMap["subOrders.$.detailedStatus"] = orderStatus
                    status = self._getStatusFromDetailedStatus(orderStatus) 
                    closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
                    if status is not None:
                        updateMap["subOrders.$.status"] = status
                    if closedStatus:
                        #if status is closed then change the paybackStatus accordingly
                        updateMap["subOrders.$.closed"] = True
                        if status == Store.ORDER_DELIVERED:
                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
                        elif status == Store.ORDER_CANCELLED:
                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
                    else:
                        closed = False
                    bulk.find(findMap).update({'$set' : updateMap})
                
            else:
                subOrdersList = self.parseMultiSubOrders(soup)
                for subOrderObj in subOrdersList:
                    subOrder =  self._isSubOrderActive(order, subOrderObj.merchantSubOrderId)
                    if subOrder is None:
                        self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrderObj])}}})
                        print "Added new suborders to Order id - ", order['orderId']
                        closed = False
                        continue
                    elif subOrder['closed']:
                        continue
                    else:
                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}
                        updateMap = {}
                        updateMap["subOrders.$.detailedStatus"] = subOrderObj.detailedStatus
                        status = self._getStatusFromDetailedStatus(orderStatus) 
                        closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
                        if status is not None:
                            updateMap["subOrders.$.status"] = status
                        if closedStatus:
                            #if status is closed then change the paybackStatus accordingly
                            updateMap["subOrders.$.closed"] = True
                            if status == Store.ORDER_DELIVERED:
                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
                            elif status == Store.ORDER_CANCELLED:
                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
                        else:
                            closed = False
                        bulk.find(findMap).update({'$set' : updateMap})
            bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
            result = bulk.execute()
            tprint(result)      
    
    def saveToAffiliate(self, offers):
        raise NotImplementedError
        
def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj