Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: Manish 

'''

from bs4 import BeautifulSoup

from bson.binary import Binary

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder, ShopCluesAffiliateInfo

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \

    Store as MStore, ungzipResponse, tprint

from dtr.storage import Mongo

from dtr.storage.Mongo import getImgSrc

from dtr.utils.utils import fetchResponseUsingProxy

from pprint import pprint

from pymongo import MongoClient

import json

import pymongo

import re

import time

import traceback

import urllib

import urllib2

from urlparse import urlparse, parse_qs

import xml.etree.ElementTree as ET

from dtr.storage import MemCache

from dtr.storage.Mongo import getDealRank

USERNAME='profittill2@gmail.com'

PASSWORD='spice@2020'

AFFLIATE_TRASACTIONS_URL = "https://admin.optimisemedia.com/v2/reports/affiliate/leads/leadsummaryexport.aspx?Contact=796881&Country=26&Agency=95&Merchant=420562&Status=-1&Year=%d&Month=%d&Day=%d&EndYear=%d&EndMonth=%d&EndDay=%d&DateType=0&Sort=CompletionDate&Login=1347562DA5E3EFF6FB1561765C47C782&Format=XML&RestrictURL=0"

ORDER_TRACK_URL='http://www.shopclues.com/index.php?dispatch=order_lookup.details'

ORDER_TRACK_URL_DB='https://sm.shopclues.com/trackOrder?'

BASE_URL= 'http://www.shopclues.com'

BASE_MURL= 'http://m.shopclues.com'

class Store(MStore):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      MStore.ORDER_PLACED : ['payment successful', 'new order - cod confirmation pending', 'processing', 'quality check','on schedule', 'processing - pickup-initiated', 'processing - ready to dispatch'],

                      MStore.ORDER_DELIVERED : ['delivered', 'complete'],

                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched','shipped','order handed to courier'],

                      MStore.ORDER_CANCELLED : ['payment failed', 'canceled', 'payment declined', 'order on hold - cancellation requested by customer', 'courier returned']

                      }

    OrderStatusConfirmationMap= {

                                 "P" : "Payment Successful",

                                 "D" : "Order Declined",

                                 "O" : "New Order - COD confirmation Pending"

                                 }

    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def convertToObj(self,offer):

        orderRef = offer['TransactionId']

        orderRef = orderRef[0:len(orderRef)-10]

        offer1 = ShopCluesAffiliateInfo(offer['UID'], offer['TransactionTime'], offer['TransactionId'], orderRef,  offer['MerchantRef'], offer['Merchant'], offer['PID'], offer['Product'], float(str(offer['SR'])), float(str(offer['TransactionValue'])), offer['UKey'], offer['ClickTime'], offer['Status'])

        return offer1

    def _saveToAffiliate(self, offers):

        collection = self.db.shopcluesOrderAffiliateInfo

        mcollection = self.db.merchantOrder

        for offerObj in offers:

            offer = self.convertToObj(offerObj)

            collection.update({"transactionId":offer.transactionId, "subTagId":offer.subTagId, "payOut":offer.payOut},{"$set":todict(offer)}, upsert=True)

            mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True}, {"$set":{"subOrders.$.missingAff":False}})

    def scrapeAffiliate(self, startDate=datetime.today() - timedelta(days=10), endDate=datetime.today()):

        uri = AFFLIATE_TRASACTIONS_URL%(startDate.year,startDate.month,startDate.day,endDate.year,endDate.month,endDate.day)

        root = ET.parse(urllib2.urlopen(uri)).getroot()

        if len(root)> 0 and len(root[0])> 0:

            offers = []

            for child in root[0][0]:

                offers.append(child.attrib)

            self._saveToAffiliate(offers)

    def _setLastSaleDate(self, saleDate):

        self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})

    def getName(self):

        return "shopclues"    

    def _getLastSaleDate(self,):

        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})

        if lastDaySaleObj is None:

            return datetime.min

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus, self.store_id

        return None

    def _getSingleSubOrderMap(self, orderId, soup, orderObj):

        orderStatus = self.OrderStatusConfirmationMap.get(orderObj['0']['status'])

        statusTime= soup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()

        productDetailsMap = {}

        orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)

        firstRow = orderTable.pop(0)

        totalColumns = len(firstRow.find_all('th'))

        jsonSubOrdersMap = {}

        count = 1

        for val in orderObj['0']['items'].values():

            newCount = 0

            counts = []

            for key in jsonSubOrdersMap.keys():

                splitKey = key.split('-')

                if str(val['child']) == splitKey[0]:

                    counts.append(int(splitKey[1]))

            if len(counts) >0:

                newCount = max(counts)

            count = newCount +1

            jsonSubOrdersMap[str(val['child'])+'-'+str(count)] = val

        print jsonSubOrdersMap.items()

        count = 1

        for orderTr in orderTable:

            productDetailsSubMap = None

            productDetailsSubMap = {}

            cols = orderTr.find_all('td')

            product_details = cols[0].find_all('a')

            #print product_details

            productUrl = product_details[0].get('href')

            productName = product_details[0].contents[0].strip()

            quantity = int(cols[1].text.strip())

            sellingPrice = long(cols[2].text.strip().replace("Rs.","").replace(',',''))

            discount = 0

            subtotal = 0

            if totalColumns == 5:

                if cols[3].text.strip()!='-' and 'Rs.' in cols[3].text.strip():

                    discount = long(cols[3].text.strip().replace("Rs.","").replace(',',''))

                else:

                    discount = 0

                subtotal = long(cols[4].text.strip().replace("Rs.","").replace(',',''))

            else:

                subtotal = long(cols[3].text.strip().replace("Rs.","").replace(',',''))

            productDetailsSubMap['productUrl']=BASE_MURL+productUrl

            productDetailsSubMap['productName']=productName

            productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL_DB+'order_id=' +str(orderId)+'&email_id='+ orderObj['0']['email']

            productDetailsSubMap['sellingPrice']=sellingPrice

            productDetailsSubMap['quantity']=quantity

            productDetailsSubMap['discount']=discount

            productDetailsSubMap['subtotal']=subtotal

            key = str(orderId)+'-'+str(count)

            print 'SubOrder Map Key--',key

            jsonSubOrderDetails = jsonSubOrdersMap.get(str(orderId)+'-'+str(count))

            productCode = jsonSubOrderDetails['product_code']

            productImgUrl = jsonSubOrderDetails['images']['image_path'][0]

            productDetailsSubMap['productCode']=productCode

            productDetailsSubMap['imgUrl']=productImgUrl

            productDetailsSubMap['subOrderStatus']=orderStatus

            productDetailsSubMap['subOrderStatusTime']=statusTime

            productDetailsMap[str(orderId)+'-'+str(count)]=productDetailsSubMap

            count = count +1

        return productDetailsMap

    def _getMultiSubOrdersMap(self, orderId, soup, orderObj):

        orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)

        firstRow = orderTable.pop(0)

        #totalColumns = len(firstRow.find_all('td'))

        productDetailsMap = {}

        '''

        jsonSubOrdersMap = {}

        for val in orderObj['0']['items'].values():

            jsonSubOrdersMap[val['order_id']] = val

        '''

        existingOrders = []

        for orderTr in orderTable:

            subOrderDetailsMap = {}

            cols = orderTr.find_all('td')

            product_details = cols[0].find_all('a')

            #print product_details

            subOrderId= product_details[1].contents[0].strip()

            if subOrderId in existingOrders:

                continue

            else:

                existingOrders.append(subOrderId)

            #productUrl = product_details[0].get('href')

            #productName = product_details[0].contents[0].strip()

            subOrderTrackingParsingUrl = product_details[1].get('href')

            #subOrderTrackingUrl = subOrderTrackingParsingUrl.split('order_lookup.details&')[1]

            '''

            quantity = int(cols[1].text.strip())

            sellingPrice = long(cols[2].text.strip().replace("Rs.",""))

            discount = 0

            subtotal = 0

            if totalColumns == 5:

                if cols[3].text.strip()!='-' or 'Rs.' in cols[3].text.strip():

                    discount = long(cols[3].text.strip().replace("Rs.",""))

                else:

                    discount = 0                    

                subtotal = long(cols[4].text.strip().replace("Rs.",""))

            else:

                subtotal = long(cols[3].text.strip().replace("Rs.",""))

            productDetailsSubMap['productUrl']=BASE_MURL+productUrl

            productDetailsSubMap['productName']=productName

            productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL_DB+subOrderTrackingUrl

            productDetailsSubMap['sellingPrice']=sellingPrice

            productDetailsSubMap['quantity']=quantity

            productDetailsSubMap['discount']=discount

            productDetailsSubMap['subtotal']=subtotal

            jsonSubOrderDetails = jsonSubOrdersMap.get(subOrderId)

            productCode = jsonSubOrderDetails['product_code']

            productImgUrl = jsonSubOrderDetails['images']['image_path'][0]

            productDetailsSubMap['productCode']=productCode

            productDetailsSubMap['imgUrl']=productImgUrl

            '''

            br1 = getBrowserObject()

            orderTrackingPage = br1.open(BASE_URL+subOrderTrackingParsingUrl)

            headers = str(orderTrackingPage.info()).split('\n')

            orderTrackingPage= ungzipResponse(orderTrackingPage)

            jsonResponse = None

            for header in headers:

                header = header.split(':')

                if header[0] == 'Content-Type' and 'json' in header[1]:

                    jsonResponse = json.loads(orderTrackingPage)

            orderTrackingPageSoup = None

            if jsonResponse is not None:

                orderTrackingPageSoup = BeautifulSoup(str(jsonResponse['text']))

            else:

                orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)

            '''

            subOrderStatusList = orderTrackingPageSoup.findAll(attrs={'class' : 'price ord_status'})

            subOrderStatus = subOrderStatusList[0].contents[0].strip()

            subOrderStatusTime= orderTrackingPageSoup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()

            productDetailsSubMap['subOrderStatus'] = subOrderStatus

            productDetailsSubMap['subOrderStatusTime'] = subOrderStatusTime

            productDetailsSubMap['parentOrderId']=orderId

            '''

            subOrderDetailsMap = self._getSingleSubOrderMap(subOrderId, orderTrackingPageSoup, orderObj)

            productDetailsMap = dict(productDetailsMap.items()+subOrderDetailsMap.items())

        return productDetailsMap

    def updateCashbackInSubOrders(self, subOrders):

        for subOrder in subOrders:

            cashbackStatus = Store.CB_NA

            cashbackAmount = 0

            percentage = 0

            amount = subOrder.amountPaid

            if amount > 0:

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)

                if cashbackAmount > 0:

                    cashbackStatus = Store.CB_PENDING

            subOrder.cashBackStatus = cashbackStatus

            subOrder.cashBackAmount = cashbackAmount

            subOrder.cashBackPercentage = percentage

        return subOrders

    def _parseOrders(self, orderId, mpOrderId, subTagId, userId, page, orderSuccessUrl, orderObj):

        soup = BeautifulSoup(page)

        productDetailsMap = {}

        paymentFields = soup.findAll(attrs={'class' : 'box_paymentcalculations_row'})

        orderDateList = soup.findAll(attrs={'class':'price ord_date'})

        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("\r","").replace("  ","")

        ordersubtotal=0

        ordercluebucks=0

        ordershippingcost=0

        ordertotal=0

        for val in paymentFields:

            for value in val.contents:

                if value is not None:

                    if 'div' in str(value).strip():

                        if 'Subtotal' in value.text.strip():

                            ordersubtotal = long(val.contents[3].text.strip().replace("Rs.","").replace(',',''))

                            print ordersubtotal

                        if 'Shipping Cost' in value.text.strip():

                            ordershippingcost = long(val.contents[3].text.strip().replace("Rs.","").replace(',',''))

                            print ordershippingcost

                        if 'Clue' in value.text.strip():

                            ordercluebucks = long(val.contents[3].text.strip().replace("Rs.","").replace(',',''))

                            print ordercluebucks

                        if 'Total' in value.text.strip():

                            ordertotal = val.contents[3].text.strip().replace("Rs.","")

                            ordertotal = ordertotal.replace(',','')

                            print ordertotal

        if orderObj['0']['is_parent_order'] == 'N':

            productDetailsMap = self._getSingleSubOrderMap(mpOrderId, soup, orderObj)

        else:

            productDetailsMap = self._getMultiSubOrdersMap(mpOrderId, soup, orderObj)

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        merchantOrder.placedOn = placedOn

        merchantOrder.merchantOrderId = orderObj['0']['order_id']

        merchantOrder.paidAmount = ordertotal

        totalOrdersAmount = 0

        totalDiscount = 0

        subOrders= []

        for key in productDetailsMap:

            subOrderDetail = productDetailsMap.get(key)

            totalOrdersAmount = totalOrdersAmount  + (subOrderDetail['sellingPrice'] * subOrderDetail['quantity'])

            totalDiscount = totalDiscount + (subOrderDetail['discount'] * subOrderDetail['quantity'])

            subOrder = SubOrder(subOrderDetail['productName'], subOrderDetail['productUrl'], placedOn, subOrderDetail['subtotal'])

            subOrder.merchantSubOrderId = key

            subOrder.detailedStatus = subOrderDetail['subOrderStatus']

            subOrder.imgUrl = subOrderDetail['imgUrl']

            subOrder.offerDiscount = subOrderDetail['discount'] * subOrderDetail['quantity']

            subOrder.unitPrice = subOrderDetail['sellingPrice']

            subOrder.productCode = subOrderDetail['productCode']

            subOrder.amountPaid = subOrderDetail['subtotal']

            subOrder.quantity = subOrderDetail['quantity']

            subOrder.tracingkUrl = subOrderDetail['subOrderTrackingUrl']

            dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

            subOrder.dealRank = dealRank.get('rank')

            subOrder.rankDesc = dealRank.get('description')

            subOrders.append(subOrder)

        print totalOrdersAmount, totalDiscount

        merchantOrder.totalAmount = totalOrdersAmount

        merchantOrder.discountApplied = totalDiscount

        merchantOrder.deliveryCharges = ordershippingcost

        merchantOrder.subOrders = self.updateCashbackInSubOrders(subOrders)

        return merchantOrder

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp = {}

        try:

            rawHtmlSoup = BeautifulSoup(rawHtml)

            emailId = None

            orderObj = None

            merchantOrderId = None

            for script in rawHtmlSoup.find_all('script'):

                if 'orderConfirmation' in script.text:

                    orderObj = json.loads(script.text.strip().split('\n')[0].split('orderConfirmation = ')[1].split(';')[0])

                    if merchantOrderId is None:

                        merchantOrderId = orderObj['0']['order_id']

                        emailId = orderObj['0']['email']

            br = getBrowserObject()

            url = ORDER_TRACK_URL +'&order_id=' +str(merchantOrderId)+'&email_id='+ emailId

            page = br.open(url)

            headers = str(page.info()).split('\n')

            page = ungzipResponse(page)

            jsonResponse = None

            for header in headers:

                header = header.split(':')

                if header[0] == 'Content-Type' and 'json' in header[1]:

                    jsonResponse = json.loads(page)

            if jsonResponse is not None:

                page = jsonResponse['text']

            merchantOrder = self._parseOrders(orderId, merchantOrderId, subTagId, userId, page, orderSuccessUrl, orderObj)

            merchantOrder.orderTrackingUrl = ORDER_TRACK_URL_DB + 'order_id=' +str(merchantOrderId)+'&email_id='+ emailId

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp

        except:

            print "Error occurred"

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

    def parseSingleSubOrder(self, soup, emailId, subOrderId):

        orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})

        subOrderStatus = orderStatusList[0].contents[0].strip()

        orderDateList = soup.findAll(attrs={'class':'price ord_date'})

        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("\r","").replace("  ","")    

        orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)

        firstRow = orderTable.pop(0)

        totalColumns = len(firstRow.find_all('th'))

        subOrders =[]

        count =1   

        for orderTr in orderTable:

            cols = orderTr.find_all('td')

            product_details = cols[0].find_all('a')

            #print product_details

            productUrl = product_details[0].get('href')

            productName = product_details[0].contents[0].strip()

            quantity = int(cols[1].text.strip())

            sellingPrice = long(cols[2].text.strip().replace("Rs.","").replace(',',''))

            discount = 0

            subtotal = 0

            if totalColumns == 5:

                if cols[3].text.strip()!='-' and 'Rs.' in cols[3].text.strip():

                    discount = long(cols[3].text.strip().replace("Rs.","").replace(',',''))

                else:

                    discount = 0

                subtotal = long(cols[4].text.strip().replace("Rs.","").replace(',',''))

            else:

                subtotal = long(cols[3].text.strip().replace("Rs.","").replace(',',''))

            br = getBrowserObject()

            productPage = br.open(BASE_MURL+productUrl)

            productPageHeaders = str(productPage.info()).split('\n')

            productPage = ungzipResponse(productPage)

            jsonProductResponse = None

            for header in productPageHeaders:

                header = header.split(':')

                if header[0] == 'Content-Type' and 'json' in header[1]:

                    jsonProductResponse = json.loads(productPage)

            productPageSoup= None

            if jsonProductResponse is not None:

                productPageSoup = BeautifulSoup(jsonProductResponse['text'])

            else:

                productPageSoup = BeautifulSoup(productPage)

            productCode = productPageSoup.find('input', {'type':'hidden'})['value']

            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})

            productImgUrl = ''

            if allproductImageTags is not None and len(allproductImageTags)>0:

                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()

            subOrder = SubOrder(productName, productUrl, placedOn, subtotal)

            subOrder.merchantSubOrderId = str(subOrderId)+'-'+str(count)

            subOrder.detailedStatus = subOrderStatus

            subOrder.imgUrl = productImgUrl

            subOrder.offerDiscount = discount*quantity

            subOrder.unitPrice = sellingPrice

            subOrder.productCode = productCode

            subOrder.amountPaid = subtotal

            subOrder.quantity = quantity

            subOrder.tracingkUrl = ORDER_TRACK_URL_DB +  'order_id=' +subOrderId+'&email_id='+ emailId

            subOrders.append(subOrder)

            count = count +1

        subOrders = self.updateCashbackInSubOrders(subOrders)

        return subOrders

    def parseMultiSubOrders(self, soup, emailId):

        orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)

        '''

        firstRow = orderTable.pop(0)

        totalColumns = len(firstRow.find_all('td'))

        orderDateList = soup.findAll(attrs={'class':'price ord_date'})

        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("  ","")

        '''    

        subOrders = []

        existingOrders = []

        for orderTr in orderTable:

            cols = orderTr.find_all('td')

            product_details = cols[0].find_all('a')

            #print product_details

            subOrderId= product_details[1].contents[0].strip()

            if subOrderId in existingOrders:

                continue

            else:

                existingOrders.append(subOrderId)

            '''

            productUrl = product_details[0].get('href')

            productName = product_details[0].contents[0].strip()

            '''

            subOrderTrackingParsingUrl = product_details[1].get('href')

            #subOrderTrackingUrl = subOrderTrackingParsingUrl.split('order_lookup.details&')[1]

            '''

            quantity = int(cols[1].text.strip())

            sellingPrice = long(cols[2].text.strip().replace("Rs.",""))

            discount = 0

            subtotal = 0

            if totalColumns == 5:

                if cols[3].text.strip()!='-' or 'Rs.' in cols[3].text.strip():

                    discount = long(cols[3].text.strip().replace("Rs.",""))

                else:

                    discount = 0

                subtotal = long(cols[4].text.strip().replace("Rs.",""))

            else:

                subtotal = long(cols[3].text.strip().replace("Rs.",""))

            br = getBrowserObject()

            productPage = br.open(BASE_MURL+productUrl)

            productPageHeaders = str(productPage.info()).split('\n')

            productPage = ungzipResponse(productPage)

            jsonProductResponse = None

            for header in productPageHeaders:

                header = header.split(':')

                if header[0] == 'Content-Type' and 'json' in header[1]:

                    jsonProductResponse = json.loads(productPage)

            productPageSoup= None

            if jsonProductResponse is not None:

                productPageSoup = BeautifulSoup(jsonProductResponse['text'])

            else:

                productPageSoup = BeautifulSoup(productPage)

            productCode = productPageSoup.find('input', {'type':'hidden'})['value']

            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})

            productImgUrl = ''

            if allproductImageTags is not None and len(allproductImageTags)>0:

                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()

            productDetailsSubMap['productCode']=productCode

            productDetailsSubMap['imgUrl']=productImgUrl

            '''

            br1 = getBrowserObject()

            orderTrackingPage = br1.open(BASE_URL+'&'+subOrderTrackingParsingUrl)

            headers = str(orderTrackingPage.info()).split('\n')

            orderTrackingPage= ungzipResponse(orderTrackingPage)

            jsonResponse = None

            for header in headers:

                header = header.split(':')

                if header[0] == 'Content-Type' and 'json' in header[1]:

                    jsonResponse = json.loads(orderTrackingPage)

            orderTrackingPageSoup = None

            if jsonResponse is not None:

                orderTrackingPageSoup = BeautifulSoup(str(jsonResponse['text']))

            else:

                orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)

            subOrdersDetails = self.parseSingleSubOrder(orderTrackingPageSoup, emailId, subOrderId)

            subOrders = list(set(subOrders + subOrdersDetails))

        return subOrders

    def scrapeStoreOrders(self,):

        #collectionMap = {'palcedOn':1}

        searchMap = {}

        collectionMap = {"orderTrackingUrl":1}

        orders = self._getActiveOrders(searchMap,collectionMap)

        for order in orders:

            print "Order", self.store_name, order['orderId'], order['orderTrackingUrl']

            url = ORDER_TRACK_URL + order['orderTrackingUrl'].split('trackOrder?')[1]

            page = fetchResponseUsingProxy(url)

            soup = BeautifulSoup(page)

            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

            closed = True

            orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})

            if orderStatusList is not None and len(orderStatusList)>0:

                subOrderId = soup.findAll(attrs={'class':'price ord_no'})[0].text.strip()

                orderStatus = orderStatusList[0].contents[0].strip()

                orderTable = soup.body.find("table", {'class':'table product-list'}).findAll('tr', recursive=False)

                orderTable.pop(0)

                count = 1

                while count <= len(orderTable):

                    subOrder =  self._isSubOrderActive(order, str(subOrderId)+'-'+str(count))

                    count = count +1

                    if subOrder is None:

                        try:

                            subOrders = self.parseSingleSubOrder(soup, order['orderTrackingUrl'].split('email_id=')[1], subOrderId)

                            for subOrder in subOrders:

                                if subOrder is None:

                                    continue

                                self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrder])}}})

                                print "Added new suborders to Order id - ", order['orderId']

                                closed = False

                        except:

                            pass

                        continue

                    elif subOrder['closed']:

                        continue

                    else:

                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}

                        updateMap = {}

                        updateMap["subOrders.$.detailedStatus"] = orderStatus

                        status = self._getStatusFromDetailedStatus(orderStatus) 

                        closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                        if status is not None:

                            updateMap["subOrders.$.status"] = status

                        if closedStatus:

                            #if status is closed then change the paybackStatus accordingly

                            updateMap["subOrders.$.closed"] = True

                            if status == Store.ORDER_DELIVERED:

                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                            elif status == Store.ORDER_CANCELLED:

                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                        else:

                            closed = False

                        bulk.find(findMap).update({'$set' : updateMap})

            else:

                subOrdersList = self.parseMultiSubOrders(soup, order['orderTrackingUrl'].split('email_id=')[1])

                for subOrderObj in subOrdersList:

                    subOrder =  self._isSubOrderActive(order, subOrderObj.merchantSubOrderId)

                    if subOrder is None:

                        self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrderObj])}}})

                        print "Added new suborders to Order id - ", order['orderId']

                        closed = False

                        continue

                    elif subOrder['closed']:

                        continue

                    else:

                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}

                        updateMap = {}

                        updateMap["subOrders.$.detailedStatus"] = subOrderObj.detailedStatus

                        status = self._getStatusFromDetailedStatus(orderStatus) 

                        closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                        if status is not None:

                            updateMap["subOrders.$.status"] = status

                        if closedStatus:

                            #if status is closed then change the paybackStatus accordingly

                            updateMap["subOrders.$.closed"] = True

                            if status == Store.ORDER_DELIVERED:

                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                            elif status == Store.ORDER_CANCELLED:

                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                        else:

                            closed = False

                        bulk.find(findMap).update({'$set' : updateMap})

            bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})

            result = bulk.execute()

            tprint(result)      

    '''

    def getTrackingUrls(self, userId):

        missingOrderUrls = ['https://sm.shopclues.com/myaccount','https://sm.shopclues.com/myorders']

        return missingOrderUrls

    def parseMyProfileForEmailId(self, userId, url, rawhtml):

        profileSoup = BeautifulSoup(rawhtml)

        if profileSoup.find('input', {'name':'user_email'}) is not None:

            emailId = profileSoup.find('input', {'name':'user_email'})['value']

            if emailId is not None and emailId.strip() !='':

                mc = MemCache()

                mc.set(str(userId), emailId, 600)

                return 'EMAIL_SET_SUCCESS'

            else:

                return 'EMAIL_NOT_FOUND'

        else:

            return 'EMAIL_NOT_FOUND'

    def parseMyOrdersForEmailId(self, userId, url, rawhtml):

        myOrdersPageSoup = BeautifulSoup(rawhtml)

        if myOrdersPageSoup.find('a', {'class':'detail'}) is not None:

            emailId = myOrdersPageSoup.find('a', {'class':'detail'})['href'].split('&')[1].split('=')[1]

            mc = MemCache()

            mc.set(str(userId), emailId, 600)

            return 'EMAIL_SET_SUCCESS'

        else:

            return 'EMAIL_NOT_FOUND'

    '''        

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj