Subversion Repositories SmartDukaan

Rev

Rev 14946 | Rev 15880 | Go to most recent revision | Blame | Compare with Previous | Last modification | View Log | RSS feed

'''
Created on Jan 15, 2015

@author: amit
'''
from bs4 import BeautifulSoup
from bson.binary import Binary
from datetime import datetime, date, timedelta
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder
from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \
    Store as MStore, ungzipResponse, tprint
from dtr.storage import Mongo
from dtr.storage.Mongo import getImgSrc, getDealRank
from dtr.utils.utils import fetchResponseUsingProxy
from pprint import pprint
from pymongo import MongoClient
from urlparse import urlparse, parse_qs
import json
import pymongo
import re
import time
import traceback
import urllib

USERNAME='profittill2@gmail.com'
PASSWORD='spice@2020'
AFFILIATE_URL='http://affiliate.snapdeal.com'
POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'
ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'
CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'

class Store(MStore):
    
    '''
    This is to map order statuses of our system to order statuses of snapdeal.
    And our statuses will change accordingly.
    
    '''
    OrderStatusMap = {
                      MStore.ORDER_PLACED : ['in progress', 'pending for verification', 'not available', 'in process', 'processing', 'processed'],
                      MStore.ORDER_DELIVERED : ['delivered'],
                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched'],
                      MStore.ORDER_CANCELLED : ['closed for vendor reallocation', 'cancelled', 'product returned by courier', 'returned', 'n/a', 'courier returned']
                      }
    
    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE
    def __init__(self,store_id):
        super(Store, self).__init__(store_id)

    def getName(self):
        return "snapdeal"
    
    def scrapeAffiliate(self, startDate=None, endDate=None):
        br = getBrowserObject()
        br.open(AFFILIATE_URL)
        br.select_form(nr=0)
        br.form['data[User][password]'] = PASSWORD 
        br.form['data[User][email]'] = USERNAME
        br.submit()
        response = br.open(CONFIG_URL)
        
        token =  re.findall('"session_token":"(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
        print token
        allOffers = self._getAllOffers(br, token)
        self._saveToAffiliate(allOffers)
    
    def _setLastSaleDate(self, saleDate):
        self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})
    
        
        
    def _getLastSaleDate(self,):
        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})
        if lastDaySaleObj is None:
            return datetime.min
        
    
    
    def _parseB(self, orderId, subTagId, userId, page, orderSuccessUrl):
        soup = BeautifulSoup(page)

        orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)
        orderDetailDiv = orderDetailContainerDivs.pop(0)
        paymentDetailDiv  = orderDetailContainerDivs.pop(0)
        subOrders = orderDetailContainerDivs
        
        placedOn = orderDetailDiv.span.text.split(':')[1].strip()
        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
        merchantOrder.placedOn = placedOn
        merchantOrder.merchantOrderId = parse_qs(urlparse(orderSuccessUrl).query)['order'][0]

        paymentDivs = paymentDetailDiv.findAll('div', recursive=False)
        paymentDivs.pop(0)
        for orderTr in paymentDivs:
            orderTrString = str(orderTr)
            if "Total Amount Paid" in orderTrString:
                amountPaid = orderTr.div.find('div', {'class':'detailBlock'}).text.strip()
                merchantOrder.paidAmount = int(re.findall(r'\d+', amountPaid)[0])
            elif "Total Amount" in orderTrString:
                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]
            elif "Delivery Charges" in orderTrString:
                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]
            elif "Discount Applied" in orderTrString:
                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]
            elif "Offer Discount" in orderTrString:
                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]
 
        merchantSubOrders = []
        for subOrderElement in subOrders:
            subOrder = self.parseSubOrderB(subOrderElement, placedOn)
            if subOrder is not None:
                dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)
                subOrder.dealRank = dealRank.get('rank')
                subOrder.rankDesc = dealRank.get('description')
                merchantSubOrders.append(subOrder)
        merchantOrder.subOrders = merchantSubOrders
        return merchantOrder
    
    def _parse(self, orderId, subTagId, userId, page, orderSuccessUrl):
        
        #page=page.decode("utf-8")
        soup = BeautifulSoup(page)
        #orderHead = soup.find(name, attrs, recursive, text)
        sections = soup.findAll("section")
        
        #print sections
        
        order = sections[1]
        orderTrs = order.findAll("tr")
        
        placedOn = str(orderTrs[0].findAll("td")[1].text)
        
        #Pop two section elements
        sections.pop(0) 
        sections.pop(0)
        subOrders = sections
        
         
        merchantSubOrders = []

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
        merchantOrder.placedOn = placedOn
        merchantOrder.merchantOrderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]
        for orderTr in orderTrs:
            orderTrString = str(orderTr)
            if "Total Amount" in orderTrString:
                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]
            elif "Delivery Charges" in orderTrString:
                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]
            elif "Discount Applied" in orderTrString:
                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]
            elif "Paid Amount" in orderTrString:
                merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]

        for subOrderElement in subOrders:
            subOrders = self.parseSubOrder(subOrderElement, placedOn)                           
            merchantSubOrders.extend(subOrders)   
        
        merchantOrder.subOrders = merchantSubOrders
        return merchantOrder
        
    def parseSubOrder(self, subOrderElement, placedOn):
        subOrders = []
        productUrl = str(subOrderElement.find("a")['href'])
        subTable = subOrderElement.find("table", {"class":"lrPad"})
        subTrs = subTable.findAll("tr")
        unitPrice=None
        offerDiscount = 0
        deliveryCharges = None
        amountPaid = None
        amount = 0
        sdCash = 0
        unitPrice = 0
        for subTr in subTrs:
            subTrString = str(subTr)
            if "Unit Price" in subTrString:
                unitPrice = int(re.findall(r'\d+', subTrString)[0])
            if "Quantity" in subTrString:
                qty = int(re.findall(r'\d+', subTrString)[0])
            elif "Offer Discount" in subTrString:
                offerDiscount +=   int(re.findall(r'\d+', subTrString)[0])
            elif "SD Cash" in subTrString:
                sdCash =   int(re.findall(r'\d+', subTrString)[0])
            elif "Delivery Charges" in subTrString:
                deliveryCharges =   int(re.findall(r'\d+', subTrString)[0])
            elif "Subtotal" in subTrString:
                if int(qty) > 0:
                    amountPaid =   int(re.findall(r'\d+', subTrString)[0])/qty
                else:
                    amountPaid =   0
        if qty>0:
            amount = unitPrice - offerDiscount - sdCash
            amount = 0 if amount < 0 else amount
                
        div1 = subOrderElement.find("div", {"class": "blk lrPad subordrs"})
        if div1 is None:
            raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
        
        for strDiv in str(div1).split("<div class=\"seperator\"></div>"):
            div = BeautifulSoup(strDiv)
            productTitle = str(subOrderElement.find("a").text)
            productUrl = "http://m.snapdeal.com/" + productUrl 
            subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

            subOrder.amountPaid = amountPaid
            subOrder.deliveryCharges = deliveryCharges
            subOrder.offerDiscount = offerDiscount
            subOrder.unitPrice = int(unitPrice)
            subOrder.productCode = re.findall(r'\d+$', productUrl)[0]
            subOrder.imgUrl = Mongo.getImgSrc(subOrder.productCode, self.store_id).get('thumbnail')
            cashbackStatus = Store.CB_NA
            cashbackAmount = 0
            percentage = 0
            if amount > 0:
                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)
                if cashbackAmount > 0:
                    cashbackStatus = Store.CB_PENDING
            subOrder.cashBackStatus = cashbackStatus
            subOrder.cashBackAmount = cashbackAmount
            subOrder.cashBackPercentage = percentage
            
            
            trackAnchor = div.find("a")   
            if trackAnchor is not None:
                subOrder.tracingkUrl = str(trackAnchor['href'])
            
            divStr = str(div)
            divStr = divStr.replace("\n","").replace("\t", "")
            
            for line in divStr.split("<br />"):
                if "Suborder ID" in line:
                    subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]   
                elif "Status" in line:
                    print line
                    subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                elif "Est. Shipping Date" in line:
                    subOrder.estimatedShippingDate = line.split(":")[1].strip()
                elif "Est. Delivery Date" in line:
                    subOrder.estimatedDeliveryDate = line.split(":")[1].strip()
                elif "Courier Name" in line:
                    subOrder.courierName = line.split(":")[1].strip()
                elif "Tracking No" in line:
                    subOrder.trackingNumber = line.split(":")[1].strip()
            subOrders.append(subOrder)
        return subOrders

    def parseSubOrderB(self, subOrderElement, placedOn):
        subOrders = []
        prodDivs = subOrderElement.findAll('div', recursive=False)
        prodDetailDiv = prodDivs[1].findAll('div', recursive=False)
        
        offerDiscount = 0
        deliveryCharges = None
        amountPaid = 0
        sdCash = 0
        unitPrice = 0

        paymentDivs = prodDivs[2].findAll('div', recursive=False)
        for paymentDiv in paymentDivs:
            strPaymentDiv = str(paymentDiv)
            if "Unit Price" in strPaymentDiv:
                try:
                    unitPrice = int(re.findall(r'\d+', strPaymentDiv)[0])
                except:
                    return None
            elif "Offer Discount" in strPaymentDiv:
                offerDiscount += int(re.findall(r'\d+', strPaymentDiv)[0])
            elif "Discount" in strPaymentDiv:
                offerDiscount += int(re.findall(r'\d+', strPaymentDiv)[0])
            elif "SD Cash" in strPaymentDiv:
                sdCash =   int(re.findall(r'\d+', strPaymentDiv)[0])
            elif "Delivery Charges" in strPaymentDiv:
                deliveryCharges =   int(re.findall(r'\d+', strPaymentDiv)[0])
            elif "Subtotal" in strPaymentDiv:
                amountPaid =   int(re.findall(r'\d+', paymentDiv.find('div', {'class':'itemPriceDetail'}).text)[0])
        
        amount = unitPrice - offerDiscount - sdCash
        
        imgDiv = prodDetailDiv[0]
        otherDiv = prodDetailDiv[1]
        productTitle = otherDiv.find('div',{'class':'orderName'}).text.strip()

        productUrl = imgDiv.a['href']
        subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)
        subOrder.merchantSubOrderId = prodDivs[0].text.split(':')[1].strip()
        subOrder.detailedStatus = otherDiv.find('div',{'class':'orderStatus'}).span.text.strip()
        deliveryStatus = otherDiv.find('div',{'class':'orderDelivery'})
        if deliveryStatus is not None:
            delString = deliveryStatus.text.strip()
            arr = delString.split(':')
            if "On" in arr[0]:
                subOrder.deliveredOn = arr[1].strip()
            elif "Exp. Delivery by" in arr[0]:
                subOrder.estimatedDeliveryDate = arr[1].strip()
            else:
                subOrder.estimatedShippingDate = arr[1].strip()

        subOrder.imgUrl = imgDiv.a.img['src']
        subOrder.productCode = re.findall(r'\d+$', productUrl)[0]
        subOrder.deliveryCharges = deliveryCharges
        subOrder.offerDiscount = offerDiscount
        subOrder.unitPrice = int(unitPrice)
        cashbackStatus = Store.CB_NA
        cashbackAmount = 0
        percentage = 0
        if amountPaid > 0:
            (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)
            if cashbackAmount > 0:
                cashbackStatus = Store.CB_PENDING
        subOrder.cashBackStatus = cashbackStatus
        subOrder.cashBackAmount = cashbackAmount
        subOrder.cashBackPercentage = percentage
        
        
        courierDet = subOrderElement.find('div', {'class':'courierDetail'})
        if courierDet is not None:
            subOrder.courierName = courierDet.span.text.strip()
        trackingDet = subOrderElement.find('div', {'class':'trackingNo'})
        if trackingDet is not None:
            subOrder.trackingUrl = trackingDet.span.a['href']
            subOrder.trackingNumber = trackingDet.span.a.text.strip()

        subOrders.append(subOrder)
        return subOrder

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
                    #print merchantOrder
        resp = {}
        try:
            br = getBrowserObject()
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
            page = br.open(url)
            page = ungzipResponse(page)
            try:
                merchantOrder = self._parseB(orderId, subTagId, userId, page, orderSuccessUrl)
            except:
                traceback.print_exc()
                merchantOrder = self._parse(orderId, subTagId, userId, page, orderSuccessUrl)
            
            merchantOrder.orderTrackingUrl = url
               
            if self._saveToOrder(todict(merchantOrder)):
                resp['result'] = 'ORDER_CREATED'
            else:
                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                
            return resp
        except:
            print "Error occurred"
            traceback.print_exc()
            resp['result'] = 'ORDER_NOT_CREATED'
            return resp
             
        
        #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)
        #soup.find(name, attrs, recursive, text)

    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, value in Store.OrderStatusMap.iteritems():
            if detailedStatus.lower() in value:
                return key
        print "Detailed Status need to be mapped", detailedStatus, self.store_id
        return None
    
    
    def scrapeStoreOrders(self,):
        #collectionMap = {'palcedOn':1}
        orders = self._getActiveOrders()
        for order in orders:
            print "Order", self.store_name, order['orderId']
            try:
                url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
                page = fetchResponseUsingProxy(url)
                #page=page.decode("utf-8")
                soup = BeautifulSoup(page)
                try:
                    self.tryBParsing(order, soup)
                except:
                    traceback.print_exc()
                    sections = soup.findAll("section")
                    orderEl = sections[1]
                    orderTrs = orderEl.findAll("tr")
                    
                    placedOn = str(orderTrs[0].findAll("td")[1].text)
                    sections.pop(0)
                    sections.pop(0)
                    
                    subOrders = sections
                    bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
                    closed = True
                    for subOrderElement in subOrders:
                        div1 = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})
                        if len(div1)<=0:
                            raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")
                        subOrder = None
                        breakFlag = False
                        for strDiv in str(div1).split("<div class=\"seperator\"></div>"):
                            div = BeautifulSoup(strDiv)
                            divStr = str(div)
                            divStr = divStr.replace("\n","").replace("\t", "")
                            updateMap = {}
                            for line in divStr.split("<br />"):
                                if "Suborder ID" in line:
                                    merchantSubOrderId = re.findall(r'\d+', line)[0]
                                    #break if suborder is inactive   
                                    subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
                                    if subOrder is None:
                                        subOrders = self.parseSubOrder(subOrderElement, placedOn)
                                        self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict(subOrders)}}})
                                        print "Added new suborders to Order id - ", order['orderId']
                                        closed = False
                                        breakFlag = True
                                        break
                                    elif subOrder['closed']:
                                        breakFlag = True
                                        break
                                    else: 
                                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
                                elif "Status :" in line:
                                    detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]
                                    updateMap["subOrders.$.detailedStatus"] = detailedStatus
                                    status = self._getStatusFromDetailedStatus(detailedStatus) 
                                    closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
                                    if status is not None:
                                        updateMap["subOrders.$.status"] = status
                                    if detailedStatus == 'Closed For Vendor Reallocation':
                                        #if it is more than 6hours mark closed.
                                        closeAt = subOrder.get("closeAt") 
                                        if closeAt is None:
                                            closeAt = datetime.now() + timedelta(hours=6)
                                            updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")
                                        else:
                                            closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")
                                            if datetime.now() > closeAt:
                                                closedStatus = True
                                            
                                            
                                    if closedStatus:
                                        #if status is closed then change the paybackStatus accordingly
                                        updateMap["subOrders.$.closed"] = True
                                        if status == Store.ORDER_DELIVERED:
                                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
                                        elif status == Store.ORDER_CANCELLED:
                                            if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                                                updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
                                            
                                    else:
                                        closed = False
                                elif "Est. Shipping Date" in line:
                                    estimatedShippingDate = line.split(":")[1].strip()
                                    updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate
                                elif "Est. Delivery Date" in line:
                                    estimatedDeliveryDate = line.split(":")[1].strip()
                                    updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate
                                elif "Courier Name" in line:
                                    courierName = line.split(":")[1].strip()
                                    updateMap["subOrders.$.courierName"] = courierName
                                elif "Tracking No" in line:
                                    trackingNumber = line.split(":")[1].strip()
                                    updateMap["subOrders.$.trackingNumber"] = trackingNumber
        
                            if breakFlag:
                                continue
                                    
                            bulk.find(findMap).update({'$set' : updateMap})
                        bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
                    result = bulk.execute()
                    tprint(result)
            except:
                tprint("Could not update " + str(order['orderId']) + "For store " + self.getName())
                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
                traceback.print_exc()                
            
    def tryBParsing(self, order, soup):
        orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)
        orderDetailDiv = orderDetailContainerDivs.pop(0)
        placedOn = orderDetailDiv.span.text.split(':')[1].strip()
        
        orderDetailContainerDivs.pop(0)
        
        subOrders = orderDetailContainerDivs
        bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
        closed = True
        for subOrderElement in subOrders:
            prodDivs = subOrderElement.findAll('div', recursive=False)
            merchantSubOrderId = prodDivs[0].text.split(':')[1].strip()
            subOrder = None
            subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
            if subOrder is None:
                try:
                    subOrder = self.parseSubOrderB(subOrderElement, placedOn)
                    if subOrder is None:
                        continue
                    self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrder])}}})
                    print "Added new suborders to Order id - ", order['orderId']
                    closed = False
                except:
                    pass
                continue
            elif subOrder['closed']:
                continue
            else: 
                prodDetailDiv = prodDivs[1].findAll('div', recursive=False)
                otherDiv = prodDetailDiv[1]
                trackBlock = subOrderElement.find('div',{'class':'trackingDetailsBlock'})
                findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
                updateMap = {}
                detailedStatus = otherDiv.find('div',{'class':'orderStatus'}).span.text.strip()
                updateMap["subOrders.$.detailedStatus"] = detailedStatus
                status = self._getStatusFromDetailedStatus(detailedStatus) 
                closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]
                if status is not None:
                    updateMap["subOrders.$.status"] = status
                if 'A New Order Placed With A Different Seller' in str(trackBlock):
                    #if it is more than 6hours mark closed.
                    closeAt = subOrder.get("closeAt") 
                    if closeAt is None:
                        closeAt = datetime.now() + timedelta(hours=6)
                        updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")
                    else:
                        closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")
                        if datetime.now() > closeAt:
                            closedStatus = True
                            updateMap["subOrders.$.status"] = Store.ORDER_CANCELLED
                        
                        
                if closedStatus:
                    #if status is closed then change the paybackStatus accordingly
                    updateMap["subOrders.$.closed"] = True
                    if status == Store.ORDER_DELIVERED:
                        if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                            updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED
                    elif status == Store.ORDER_CANCELLED:
                        if subOrder.get("cashBackStatus") == Store.CB_PENDING:
                            updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED
                        
                else:
                    closed = False
                    
                deliveryStatus = otherDiv.find('div',{'class':'orderDelivery'})
                if deliveryStatus is not None:
                    delString = deliveryStatus.text.strip()
                    arr = delString.split(':')    
                    if "On" in arr[0]:
                        updateMap['subOrders.$.deliveredOn'] = arr[1].strip()
                    elif "Exp. Delivery by" in arr[0]:
                        updateMap['subOrders.$.estimatedDeliveryDate'] = arr[1].strip()
                    else:
                        updateMap['subOrders.$.estimatedShippingDate'] = arr[1].strip()
                courierDet = subOrderElement.find('div', {'class':'courierDetail'})
                if courierDet is not None:
                    updateMap['subOrders.$.courierName'] = courierDet.span.text.strip()
                trackingDet = subOrderElement.find('div', {'class':'trackingNo'})
                if trackingDet is not None:
                    updateMap['subOrders.$.trackingUrl'] = trackingDet.span.a['href']
                    updateMap['subOrders.$.trackingNumber'] = trackingDet.span.a.text.strip()                    
            bulk.find(findMap).update({'$set' : updateMap})
        bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})
        result = bulk.execute()
        tprint(result)

    
    def _saveToAffiliate(self, offers):
        collection = self.db.snapdealOrderAffiliateInfo
        mcollection = self.db.merchantOrder
        for offer in offers:
            offer = self.covertToObj(offer)
            collection.update({"adId":offer.adId, "saleAmount":offer.saleAmount, "payOut":offer.payOut},{"$set":todict(offer)}, upsert=True)
            mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True}, {"$set":{"subOrders.$.missingAff":False}})
            
    
    def _getAllOffers(self, br, token):
        allOffers = []
        nextPage = 1  
        while True:
            data = getPostData(token, nextPage)
            response = br.open(POST_URL, data)
            rmap = json.loads(ungzipResponse(response))
            if rmap is not None:
                rmap = rmap['response']
                print rmap
                if rmap is not None and len(rmap['errors'])==0:
                    allOffers += rmap['data']['data']
            nextPage += 1
            if rmap['data']['pageCount']<nextPage:
                break
        
        return allOffers
    
    def covertToObj(self,offer):
        offerData = offer['Stat']
        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 
                              offerData['datetime'], int(float(offerData['payout'])), offer['Offer']['name'], offerData['ip'], int(float(offerData['conversion_sale_amount'])))
        offer1.saleTime = int(time.mktime(datetime.strptime(offer1.saleDate, "%Y-%m-%d %H:%M:%S").timetuple()))
        return offer1
def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):
    endDate=date.today() + timedelta(days=1)
    startDate=endDate - timedelta(days=31)

    parameters = (
        ("page",str(page)),
        ("limit",str(limit)),
        ("fields[]","Stat.offer_id"),
        ("fields[]","Stat.datetime"),
        ("fields[]","Offer.name"),
        ("fields[]","Stat.conversion_status"),
        ("fields[]","Stat.conversion_sale_amount"),
        ("fields[]","Stat.payout"),
        ("fields[]","Stat.ip"),
        ("fields[]","Stat.ad_id"),
        ("fields[]","Stat.affiliate_info1"),
        ("sort[Stat.datetime]","desc"),
        ("filters[Stat.date][conditional]","BETWEEN"),
        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),
        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),
        ("data_start",startDate.strftime('%Y-%m-%d')),
        ("data_end",endDate.strftime('%Y-%m-%d')),
        ("Method","getConversions"),
        ("NetworkId","jasper"),
        ("SessionToken",token),
    )
    #Encode the parameters
    return urllib.urlencode(parameters)

def main():
    #print todict([1,2,"3"])
    store = getStore(3)
    store.parseOrderRawHtml(3322211, "3232311", 2, "32311243", "https://m.snapdeal.com/purchaseMobileComplete?code=926317148e560dbbe07f3326332662ec&order=6139907711")
    #store.scrapeStoreOrders()
    #store._isSubOrderActive(8, "5970688907")
    #store.scrapeAffiliate()
    #store.scrapeStoreOrders()


            
def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj

if __name__ == '__main__':
    main()