Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: Manish 

'''

from bs4 import BeautifulSoup

from bson.binary import Binary

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder, ShopCluesAffiliateInfo

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \

    Store as MStore, ungzipResponse, tprint

from dtr.storage import Mongo

from dtr.storage.Mongo import getImgSrc

from dtr.utils.utils import fetchResponseUsingProxy

from pprint import pprint

from pymongo import MongoClient

import json

import pymongo

import re

import time

import traceback

import urllib

import urllib2

from urlparse import urlparse, parse_qs

import xml.etree.ElementTree as ET

from dtr.storage import MemCache

from dtr.storage.Mongo import getDealRank

USERNAME='profittill2@gmail.com'

PASSWORD='spice@2020'

AFFLIATE_TRASACTIONS_URL = "https://admin.optimisemedia.com/v2/reports/affiliate/leads/leadsummaryexport.aspx?Contact=796881&Country=26&Agency=95&Merchant=420562&Status=-1&Year=%d&Month=%d&Day=%d&EndYear=%d&EndMonth=%d&EndDay=%d&DateType=0&Sort=CompletionDate&Login=1347562DA5E3EFF6FB1561765C47C782&Format=XML&RestrictURL=0"

ORDER_TRACK_URL='http://www.shopclues.com/index.php?dispatch=order_lookup.details'

ORDER_TRACK_URL_DB='https://sm.shopclues.com/trackOrder?'

BASE_URL= 'http://www.shopclues.com'

BASE_MURL= 'http://m.shopclues.com'

class Store(MStore):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      MStore.ORDER_PLACED : ['payment successful', 'new order - cod confirmation pending', 'processing', 'quality check','on schedule', 'processing - pickup initiated', 'processing - ready to dispatch','processing - procurement delay from merchant','processing - slight procurment delay from merchant','cod order confirmed by customer'],

                      MStore.ORDER_DELIVERED : ['delivered', 'complete'],

                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched','shipped','order handed to courier','order handed over to courier'],

                      MStore.ORDER_CANCELLED : ['payment failed', 'canceled', 'payment declined', 'order on hold - cancellation requested by customer', 'courier returned', 'canceled on customer request', 'canceled by customer','order canceled by customer','canceled - address not shippable']

                      }

    OrderStatusConfirmationMap= {

                                 "P" : "Payment Successful",

                                 "D" : "Order Declined",

                                 "O" : "New Order - COD confirmation Pending"

                                 }

    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def convertToObj(self,offer):

        orderRef = offer['MerchantRef']

        if len(orderRef)>15:

            orderRef = orderRef[0:len(orderRef)-10]

        offer1 = ShopCluesAffiliateInfo(offer['UID'], offer['TransactionTime'], offer['TransactionId'], orderRef,  orderRef, offer['Merchant'], offer['PID'], offer['Product'], float(str(offer['SR'])), float(str(offer['TransactionValue'])), offer['UKey'], offer['ClickTime'], offer['Status'])

        return offer1

    def _saveToAffiliate(self, offers):

        collection = self.db.shopcluesOrderAffiliateInfo

        mcollection = self.db.merchantOrder

        for offerObj in offers:

            offer = self.convertToObj(offerObj)

            collection.update({"transactionId":offer.transactionId, "subTagId":offer.subTagId, "payOut":offer.payOut},{"$set":todict(offer)}, upsert=True)

            mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True}, {"$set":{"subOrders.$.missingAff":False}})

    def scrapeAffiliate(self, startDate=datetime.today() - timedelta(days=10), endDate=datetime.today()):

        uri = AFFLIATE_TRASACTIONS_URL%(startDate.year,startDate.month,startDate.day,endDate.year,endDate.month,endDate.day)

        root = ET.parse(urllib2.urlopen(uri)).getroot()

        if len(root)> 0 and len(root[0])> 0:

            offers = []

            for child in root[0][0]:

                offers.append(child.attrib)

            self._saveToAffiliate(offers)

    def _setLastSaleDate(self, saleDate):

        self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})

    def getName(self):

        return "shopclues"    

    def _getLastSaleDate(self,):

        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})

        if lastDaySaleObj is None:

            return datetime.min

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus, self.store_id

        return None

    def _getSingleSubOrderMap(self, orderId, soup, orderObj):

        orderStatus = self.OrderStatusConfirmationMap.get(orderObj['0']['status'])

        statusTime= soup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()

        productDetailsMap = {}

        orderTables = soup.body.findAll("table", {'class':'table product-list'})

        orderTable = orderTables[len(orderTables)-1].findAll('tr', recursive=False)

        firstRow = orderTable.pop(0)

        totalColumns = len(firstRow.find_all('th'))

        jsonSubOrdersMap = {}

        count = 1

        for val in orderObj['0']['items'].values():

            newCount = 0

            counts = []

            for key in jsonSubOrdersMap.keys():

                splitKey = key.split('-')

                if orderObj['0']['is_parent_order'] == 'N':

                    if str(val['order_id']) == splitKey[0]:

                        counts.append(int(splitKey[1]))

                else:

                    if str(val['child']) == splitKey[0]:

                        counts.append(int(splitKey[1]))

            if len(counts) >0:

                newCount = max(counts)

            count = newCount +1

            if orderObj['0']['is_parent_order'] == 'N':

                jsonSubOrdersMap[str(val['order_id'])+'-'+str(count)] = val

            else:

                jsonSubOrdersMap[str(val['child'])+'-'+str(count)] = val

        print jsonSubOrdersMap.items()

        count = 1

        for orderTr in orderTable:

            productDetailsSubMap = None

            productDetailsSubMap = {}

            cols = orderTr.find_all('td')

            product_details = cols[0].find_all('a')

            #print product_details

            productUrl = product_details[0].get('href')

            productName = product_details[0].contents[0].strip()

            quantity = int(cols[1].text.strip())

            sellingPrice = long(cols[2].text.strip().replace("Rs.","").replace(',',''))

            discount = 0

            subtotal = 0

            if totalColumns == 5:

                if cols[3].text.strip()!='-' and 'Rs.' in cols[3].text.strip():

                    discount = long(cols[3].text.strip().replace("Rs.","").replace(',',''))

                else:

                    discount = 0

                subtotal = long(cols[4].text.strip().replace("Rs.","").replace(',',''))

            else:

                subtotal = long(cols[3].text.strip().replace("Rs.","").replace(',',''))

            productDetailsSubMap['productUrl']=BASE_MURL+productUrl

            productDetailsSubMap['productName']=productName

            productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL_DB+'order_id=' +str(orderId)+'&email_id='+ orderObj['0']['email']

            productDetailsSubMap['sellingPrice']=sellingPrice

            productDetailsSubMap['quantity']=quantity

            productDetailsSubMap['discount']=discount

            productDetailsSubMap['subtotal']=subtotal

            key = str(orderId)+'-'+str(count)

            print 'SubOrder Map Key--',key

            jsonSubOrderDetails = jsonSubOrdersMap.get(str(orderId)+'-'+str(count))

            productCode = jsonSubOrderDetails['product_code']

            productImgUrl = jsonSubOrderDetails['images']['image_path'][0]

            productDetailsSubMap['productCode']=productCode

            productDetailsSubMap['imgUrl']=productImgUrl

            productDetailsSubMap['subOrderStatus']=orderStatus

            productDetailsSubMap['subOrderStatusTime']=statusTime

            productDetailsMap[str(orderId)+'-'+str(count)]=productDetailsSubMap

            count = count +1

        return productDetailsMap

    def _getMultiSubOrdersMap(self, orderId, soup, orderObj):

        orderTables = soup.body.findAll("table", {'class':'table product-list'})

        orderTable = orderTables[len(orderTables)-1].findAll('tr', recursive=False)

        firstRow = orderTable.pop(0)

        #totalColumns = len(firstRow.find_all('td'))

        productDetailsMap = {}

        '''

        jsonSubOrdersMap = {}

        for val in orderObj['0']['items'].values():

            jsonSubOrdersMap[val['order_id']] = val

        '''

        existingOrders = []

        for orderTr in orderTable:

            subOrderDetailsMap = {}

            cols = orderTr.find_all('td')

            product_details = cols[0].find_all('a')

            #print product_details

            subOrderId= product_details[1].contents[0].strip()

            if subOrderId in existingOrders:

                continue

            else:

                existingOrders.append(subOrderId)

            #productUrl = product_details[0].get('href')

            #productName = product_details[0].contents[0].strip()

            subOrderTrackingParsingUrl = product_details[1].get('href')

            #subOrderTrackingUrl = subOrderTrackingParsingUrl.split('order_lookup.details&')[1]

            '''

            quantity = int(cols[1].text.strip())

            sellingPrice = long(cols[2].text.strip().replace("Rs.",""))

            discount = 0

            subtotal = 0

            if totalColumns == 5:

                if cols[3].text.strip()!='-' or 'Rs.' in cols[3].text.strip():

                    discount = long(cols[3].text.strip().replace("Rs.",""))

                else:

                    discount = 0                    

                subtotal = long(cols[4].text.strip().replace("Rs.",""))

            else:

                subtotal = long(cols[3].text.strip().replace("Rs.",""))

            productDetailsSubMap['productUrl']=BASE_MURL+productUrl

            productDetailsSubMap['productName']=productName

            productDetailsSubMap['subOrderTrackingUrl']=ORDER_TRACK_URL_DB+subOrderTrackingUrl

            productDetailsSubMap['sellingPrice']=sellingPrice

            productDetailsSubMap['quantity']=quantity

            productDetailsSubMap['discount']=discount

            productDetailsSubMap['subtotal']=subtotal

            jsonSubOrderDetails = jsonSubOrdersMap.get(subOrderId)

            productCode = jsonSubOrderDetails['product_code']

            productImgUrl = jsonSubOrderDetails['images']['image_path'][0]

            productDetailsSubMap['productCode']=productCode

            productDetailsSubMap['imgUrl']=productImgUrl

            '''

            orderTrackingPage = fetchResponseUsingProxy(BASE_URL+subOrderTrackingParsingUrl)

            orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)

            '''

            subOrderStatusList = orderTrackingPageSoup.findAll(attrs={'class' : 'price ord_status'})

            subOrderStatus = subOrderStatusList[0].contents[0].strip()

            subOrderStatusTime= orderTrackingPageSoup.findAll(attrs={'class' : 'sts no_mobile'})[1].contents[2].strip()

            productDetailsSubMap['subOrderStatus'] = subOrderStatus

            productDetailsSubMap['subOrderStatusTime'] = subOrderStatusTime

            productDetailsSubMap['parentOrderId']=orderId

            '''

            subOrderDetailsMap = self._getSingleSubOrderMap(subOrderId, orderTrackingPageSoup, orderObj)

            productDetailsMap = dict(productDetailsMap.items()+subOrderDetailsMap.items())

        return productDetailsMap

    def updateCashbackInSubOrders(self, subOrders):

        for subOrder in subOrders:

            cashbackStatus = Store.CB_NA

            cashbackAmount = 0

            percentage = 0

            amount = subOrder.amountPaid

            if amount > 0:

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)

                if cashbackAmount > 0:

                    cashbackStatus = Store.CB_PENDING

            subOrder.cashBackStatus = cashbackStatus

            subOrder.cashBackAmount = cashbackAmount

            subOrder.cashBackPercentage = percentage

        return subOrders

    def _parseOrders(self, orderId, mpOrderId, subTagId, userId, page, orderSuccessUrl, orderObj):

        soup = BeautifulSoup(page)

        productDetailsMap = {}

        paymentFields = soup.findAll(attrs={'class' : 'box_paymentcalculations_row'})

        orderDateList = soup.findAll(attrs={'class':'price ord_date'})

        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("\r","").replace("  ","")

        ordersubtotal=0

        ordercluebucks=0

        ordershippingcost=0

        ordertotal=0

        for val in paymentFields:

            for value in val.contents:

                if value is not None:

                    if 'div' in str(value).strip():

                        if 'Subtotal' in value.text.strip():

                            ordersubtotal = long(val.contents[3].text.strip().replace("Rs.","").replace(',',''))

                            print 'Order SubTotal:- ',ordersubtotal

                        if 'Shipping Cost' in value.text.strip():

                            ordershippingcost = long(val.contents[3].text.strip().replace("Rs.","").replace(',',''))

                            print 'Shipping Cost:- ',ordershippingcost

                        if 'Clue' in value.text.strip():

                            ordercluebucks = long(val.contents[3].text.strip().replace("Rs.","").replace(',',''))

                            print 'Clue bucks:- ',ordercluebucks

                        if 'Total' in value.text.strip():

                            ordertotal = val.contents[3].text.strip().replace("Rs.","")

                            ordertotal = ordertotal.replace(',','')

                            print 'Order Total:- ',ordertotal

        if orderObj['0']['is_parent_order'] == 'N':

            productDetailsMap = self._getSingleSubOrderMap(mpOrderId, soup, orderObj)

        else:

            productDetailsMap = self._getMultiSubOrdersMap(mpOrderId, soup, orderObj)

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        merchantOrder.placedOn = placedOn

        merchantOrder.merchantOrderId = orderObj['0']['order_id']

        merchantOrder.paidAmount = ordertotal

        totalOrdersAmount = 0

        totalDiscount = 0

        subOrders= []

        for key in productDetailsMap:

            subOrderDetail = productDetailsMap.get(key)

            totalOrdersAmount = totalOrdersAmount  + (subOrderDetail['sellingPrice'] * subOrderDetail['quantity'])

            totalDiscount = totalDiscount + (subOrderDetail['discount'] * subOrderDetail['quantity'])

            subOrder = SubOrder(subOrderDetail['productName'], subOrderDetail['productUrl'], placedOn, subOrderDetail['subtotal'])

            subOrder.merchantSubOrderId = key

            subOrder.detailedStatus = subOrderDetail['subOrderStatus']

            subOrder.imgUrl = subOrderDetail['imgUrl']

            subOrder.offerDiscount = subOrderDetail['discount'] * subOrderDetail['quantity']

            subOrder.unitPrice = subOrderDetail['sellingPrice']

            subOrder.productCode = subOrderDetail['productCode']

            subOrder.amountPaid = subOrderDetail['subtotal']

            subOrder.quantity = subOrderDetail['quantity']

            subOrder.tracingkUrl = subOrderDetail['subOrderTrackingUrl']

            dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

            subOrder.dealRank = dealRank.get('rank')

            subOrder.rankDesc = dealRank.get('description')

            subOrders.append(subOrder)

        merchantOrder.totalAmount = totalOrdersAmount

        merchantOrder.discountApplied = totalDiscount

        merchantOrder.deliveryCharges = ordershippingcost

        merchantOrder.subOrders = self.updateCashbackInSubOrders(subOrders)

        return merchantOrder

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp = {}

        try:

            rawHtmlSoup = BeautifulSoup(rawHtml)

            emailId = None

            orderObj = None

            merchantOrderId = None

            for script in rawHtmlSoup.find_all('script'):

                if 'orderConfirmation' in script.text:

                    print script.text.strip().split('\n')[0].split('orderConfirmation = ')[1].split(';')[0]

                    orderObj = json.loads(script.text.strip().split('\n')[0].split('orderConfirmation = ')[1].split(';')[0])

                    if merchantOrderId is None:

                        merchantOrderId = orderObj['0']['order_id']

                        emailId = orderObj['0']['email']

            if orderObj is None or merchantOrderId is None:

                orderIdDiv = rawHtmlSoup.body.find("div", {'class':'conf_succes'})

                if orderIdDiv is None:

                    orderIdArt = rawHtmlSoup.body.find("article", {'class':'white'})

                    if orderIdArt is None:

                        resp['result'] = 'ORDER_NOT_CREATED_UNKNOWN'

                        return resp

                    else:

                        orderIdDiv= orderIdArt.find('p', recursive=False)

                        orderIdVal = str(orderIdDiv.text.split(' : ')[1])

                        print "Opening Shopclues Login Page"

                        login_url = "https://sm.shopclues.com/login"

                        br1 = login(login_url)

                        orderDetailPage = br1.open("https://sm.shopclues.com/orderconfirmation?order_id="+orderIdVal+"&status=P")

                        orderDetailPage = ungzipResponse(orderDetailPage)

                        rawHtmlSoup = BeautifulSoup(orderDetailPage)

                        for script in rawHtmlSoup.find_all('script'):

                            if 'orderConfirmation' in script.text:

                                orderObj = json.loads(script.text.strip().split('\n')[0].split('orderConfirmation = ')[1].split(';')[0])

                                if merchantOrderId is None:

                                    merchantOrderId = orderObj['0']['order_id']

                                    emailId = orderObj['0']['email']

                        logoutpage = br1.open("https://sm.shopclues.com/logout")

                        print br1.geturl()

                else:

                    orderIdDiv= orderIdDiv.find('div', recursive=False)

                    orderIdVal = str(orderIdDiv.span.text)

                    print "Opening Shopclues Login Page"

                    login_url = "https://sm.shopclues.com/login"

                    br1 = login(login_url)

                    orderDetailPage = br1.open("https://sm.shopclues.com/orderconfirmation?order_id="+orderIdVal+"&status=P")

                    orderDetailPage = ungzipResponse(orderDetailPage)

                    rawHtmlSoup = BeautifulSoup(orderDetailPage)

                    for script in rawHtmlSoup.find_all('script'):

                        if 'orderConfirmation' in script.text:

                            orderObj = json.loads(script.text.strip().split('\n')[0].split('orderConfirmation = ')[1].split(';')[0])

                            if merchantOrderId is None:

                                merchantOrderId = orderObj['0']['order_id']

                                emailId = orderObj['0']['email']

                    logoutpage = br1.open("https://sm.shopclues.com/logout")

                    print br1.geturl()

            br = getBrowserObject()

            url = ORDER_TRACK_URL +'&order_id=' +str(merchantOrderId)+'&email_id='+ emailId

            page = br.open(url)

            headers = str(page.info()).split('\n')

            page = ungzipResponse(page)

            jsonResponse = None

            for header in headers:

                header = header.split(':')

                if header[0] == 'Content-Type' and 'json' in header[1]:

                    jsonResponse = json.loads(page)

            if jsonResponse is not None:

                page = jsonResponse['text']

            merchantOrder = self._parseOrders(orderId, merchantOrderId, subTagId, userId, page, orderSuccessUrl, orderObj)

            merchantOrder.orderTrackingUrl = ORDER_TRACK_URL_DB + 'order_id=' +str(merchantOrderId)+'&email_id='+ emailId

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp

        except:

            print "Error occurred"

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            return resp

    def parseSingleSubOrder(self, soup, emailId, subOrderId):

        orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})

        subOrderStatus = orderStatusList[0].contents[0].strip()

        orderDateList = soup.findAll(attrs={'class':'price ord_date'})

        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("\r","").replace("  ","")

        orderTables = soup.body.findAll("table", {'class':'table product-list'})

        orderTable = orderTables[len(orderTables)-1].findAll('tr', recursive=False)    

        firstRow = orderTable.pop(0)

        totalColumns = len(firstRow.find_all('th'))

        subOrders =[]

        count =1   

        for orderTr in orderTable:

            cols = orderTr.find_all('td')

            product_details = cols[0].find_all('a')

            #print product_details

            productUrl = product_details[0].get('href')

            productName = product_details[0].contents[0].strip()

            quantity = int(cols[1].text.strip())

            sellingPrice = long(cols[2].text.strip().replace("Rs.","").replace(',',''))

            discount = 0

            subtotal = 0

            if totalColumns == 5:

                if cols[3].text.strip()!='-' and 'Rs.' in cols[3].text.strip():

                    discount = long(cols[3].text.strip().replace("Rs.","").replace(',',''))

                else:

                    discount = 0

                subtotal = long(cols[4].text.strip().replace("Rs.","").replace(',',''))

            else:

                subtotal = long(cols[3].text.strip().replace("Rs.","").replace(',',''))

            productPage = fetchResponseUsingProxy(BASE_MURL+productUrl)

            productPageSoup = BeautifulSoup(productPage)

            productCode = productPageSoup.find('input', {'type':'hidden'})['value']

            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})

            productImgUrl = ''

            if allproductImageTags is not None and len(allproductImageTags)>0:

                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()

            subOrder = SubOrder(productName, productUrl, placedOn, subtotal)

            subOrder.merchantSubOrderId = str(subOrderId)+'-'+str(count)

            subOrder.detailedStatus = subOrderStatus

            subOrder.imgUrl = productImgUrl

            subOrder.offerDiscount = discount*quantity

            subOrder.unitPrice = sellingPrice

            subOrder.productCode = productCode

            subOrder.amountPaid = subtotal

            subOrder.quantity = quantity

            subOrder.tracingkUrl = ORDER_TRACK_URL_DB +  'order_id=' +subOrderId+'&email_id='+ emailId

            subOrders.append(subOrder)

            count = count +1

        subOrders = self.updateCashbackInSubOrders(subOrders)

        return subOrders

    def parseMultiSubOrders(self, soup, emailId):

        orderTables = soup.body.findAll("table", {'class':'table product-list'})

        orderTable = orderTables[len(orderTables)-1].findAll('tr', recursive=False)

        firstRow = orderTable.pop(0)

        '''

        totalColumns = len(firstRow.find_all('td'))

        orderDateList = soup.findAll(attrs={'class':'price ord_date'})

        placedOn= orderDateList[0].text.strip().replace("\t","").replace("\n","").replace("  ","")

        '''    

        subOrders = []

        existingOrders = []

        for orderTr in orderTable:

            cols = orderTr.find_all('td')

            product_details = cols[0].find_all('a')

            #print product_details

            subOrderId= product_details[1].contents[0].strip()

            if subOrderId in existingOrders:

                continue

            else:

                existingOrders.append(subOrderId)

            '''

            productUrl = product_details[0].get('href')

            productName = product_details[0].contents[0].strip()

            '''

            subOrderTrackingParsingUrl = product_details[1].get('href')

            #subOrderTrackingUrl = subOrderTrackingParsingUrl.split('order_lookup.details&')[1]

            '''

            quantity = int(cols[1].text.strip())

            sellingPrice = long(cols[2].text.strip().replace("Rs.",""))

            discount = 0

            subtotal = 0

            if totalColumns == 5:

                if cols[3].text.strip()!='-' or 'Rs.' in cols[3].text.strip():

                    discount = long(cols[3].text.strip().replace("Rs.",""))

                else:

                    discount = 0

                subtotal = long(cols[4].text.strip().replace("Rs.",""))

            else:

                subtotal = long(cols[3].text.strip().replace("Rs.",""))

            br = getBrowserObject()

            productPage = br.open(BASE_MURL+productUrl)

            productPageHeaders = str(productPage.info()).split('\n')

            productPage = ungzipResponse(productPage)

            jsonProductResponse = None

            for header in productPageHeaders:

                header = header.split(':')

                if header[0] == 'Content-Type' and 'json' in header[1]:

                    jsonProductResponse = json.loads(productPage)

            productPageSoup= None

            if jsonProductResponse is not None:

                productPageSoup = BeautifulSoup(jsonProductResponse['text'])

            else:

                productPageSoup = BeautifulSoup(productPage)

            productCode = productPageSoup.find('input', {'type':'hidden'})['value']

            allproductImageTags = productPageSoup.findAll(attrs={'class' : 'pd-image'})

            productImgUrl = ''

            if allproductImageTags is not None and len(allproductImageTags)>0:

                productImgUrl= allproductImageTags[0].get('style').split("background:url('")[1].split("')no-repeat center")[0].strip()

            productDetailsSubMap['productCode']=productCode

            productDetailsSubMap['imgUrl']=productImgUrl

            '''

            orderTrackingPage = fetchResponseUsingProxy(BASE_URL+subOrderTrackingParsingUrl)

            orderTrackingPageSoup = BeautifulSoup(orderTrackingPage)

            subOrdersDetails = self.parseSingleSubOrder(orderTrackingPageSoup, emailId, subOrderId)

            subOrders = list(set(subOrders + subOrdersDetails))

        return subOrders

    def scrapeStoreOrders(self,):

        #collectionMap = {'palcedOn':1}

        searchMap = {}

        collectionMap = {"orderTrackingUrl":1}

        orders = self._getActiveOrders(searchMap,collectionMap)

        for order in orders:

            print "Order", self.store_name, order['orderId'], order['orderTrackingUrl']

            url = ORDER_TRACK_URL +'&'+ order['orderTrackingUrl'].split('trackOrder?')[1]

            print url

            page = None

            retry = 1

            while retry <=3: 

                try: 

                    page = fetchResponseUsingProxy(url)

                    break

                except:

                    traceback.print_exc()

                    retry = retry + 1

            soup = BeautifulSoup(page)

            bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

            closed = True

            orderIdSpan = soup.body.find("span", {'class':'price ord_no'})

            if orderIdSpan is not None:

                orderStatusList = soup.findAll(attrs={'class' : 'price ord_status'})

                if orderStatusList is not None and len(orderStatusList)>0:

                    subOrderId = soup.findAll(attrs={'class':'price ord_no'})[0].text.strip()

                    orderStatus = orderStatusList[0].contents[0].strip()

                    orderTables = soup.body.findAll("table", {'class':'table product-list'})

                    orderTable = orderTables[len(orderTables)-1].findAll('tr', recursive=False)

                    orderTable.pop(0)

                    count = 1

                    while count <= len(orderTable):

                        subbulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                        print 'Sub Order Id', str(subOrderId)+'-'+str(count)

                        subOrder =  self._isSubOrderActive(order, str(subOrderId)+'-'+str(count))

                        if subOrder is None:

                            try:

                                subOrders = self.parseSingleSubOrder(soup, order['orderTrackingUrl'].split('email_id=')[1], subOrderId)

                                for subOrder in subOrders:

                                    if subOrder is None:

                                        continue

                                    self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrder])}}})

                                    print "Added new suborders to Order id - ", order['orderId']

                                    closed = False

                            except:

                                traceback.print_exc()

                                pass

                            count = count+1

                            continue

                        elif subOrder['closed']:

                            count = count+1

                            continue

                        else:

                            findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": str(subOrderId)+'-'+str(count)}

                            updateMap = {}

                            updateMap["subOrders.$.detailedStatus"] = orderStatus

                            status = self._getStatusFromDetailedStatus(orderStatus) 

                            closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                            if status is not None:

                                updateMap["subOrders.$.status"] = status

                            if closedStatus:

                                #if status is closed then change the paybackStatus accordingly

                                print 'Order Closed'

                                updateMap["subOrders.$.closed"] = True

                                if status == Store.ORDER_DELIVERED:

                                    if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                        updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                                elif status == Store.ORDER_CANCELLED:

                                    if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                        updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                            else:

                                closed = False

                                print 'Order not Closed'

                            subbulk.find(findMap).update({'$set' : updateMap})

                            subresult = subbulk.execute()

                            tprint(subresult)

                        count = count +1

                    bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})

                    result = bulk.execute()

                    tprint(result)

                else:

                    subOrdersList = self.parseMultiSubOrders(soup, order['orderTrackingUrl'].split('email_id=')[1])

                    for subOrderObj in subOrdersList:

                        subbulk1 = self.db.merchantOrder.initialize_ordered_bulk_op()

                        subOrder =  self._isSubOrderActive(order, subOrderObj.merchantSubOrderId)

                        if subOrder is None:

                            self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrderObj])}}})

                            print "Added new suborders to Order id - ", order['orderId']

                            closed = False

                            continue

                        elif subOrder['closed']:

                            continue

                        else:

                            findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": subOrderId}

                            updateMap = {}

                            updateMap["subOrders.$.detailedStatus"] = subOrderObj.detailedStatus

                            status = self._getStatusFromDetailedStatus(orderStatus) 

                            closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                            if status is not None:

                                updateMap["subOrders.$.status"] = status

                            if closedStatus:

                                #if status is closed then change the paybackStatus accordingly

                                updateMap["subOrders.$.closed"] = True

                                if status == Store.ORDER_DELIVERED:

                                    if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                        updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                                elif status == Store.ORDER_CANCELLED:

                                    if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                        updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                            else:

                                closed = False

                            subbulk1.find(findMap).update({'$set' : updateMap})

                            subresult1 = subbulk1.execute()

                            tprint(subresult1)

                    bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})

                    result = bulk.execute()

                    tprint(result)

            else:

                print 'Soup Object not found for this Order', order['orderId'], order['orderTrackingUrl']

                continue      

    '''

    def getTrackingUrls(self, userId):

        missingOrderUrls = ['https://sm.shopclues.com/myaccount','https://sm.shopclues.com/myorders']

        return missingOrderUrls

    def parseMyProfileForEmailId(self, userId, url, rawhtml):

        profileSoup = BeautifulSoup(rawhtml)

        if profileSoup.find('input', {'name':'user_email'}) is not None:

            emailId = profileSoup.find('input', {'name':'user_email'})['value']

            if emailId is not None and emailId.strip() !='':

                mc = MemCache()

                mc.set(str(userId), emailId, 600)

                return 'EMAIL_SET_SUCCESS'

            else:

                return 'EMAIL_NOT_FOUND'

        else:

            return 'EMAIL_NOT_FOUND'

    def parseMyOrdersForEmailId(self, userId, url, rawhtml):

        myOrdersPageSoup = BeautifulSoup(rawhtml)

        if myOrdersPageSoup.find('a', {'class':'detail'}) is not None:

            emailId = myOrdersPageSoup.find('a', {'class':'detail'})['href'].split('&')[1].split('=')[1]

            mc = MemCache()

            mc.set(str(userId), emailId, 600)

            return 'EMAIL_SET_SUCCESS'

        else:

            return 'EMAIL_NOT_FOUND'

    '''        

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj

def login(url):

    br = getBrowserObject()

    br.set_proxies({"http": "us-il.proxymesh.com:31280"})

    br.open(url)

    response = br.open(url)

    ungzipResponseBr(response, br)

    #html = response.read()

    #print html

    br.select_form(nr=0)

    br.form['user'] = "imanthetester@gmail.com"

    br.form['password'] = "$Dl8uk"

    response = br.submit()

    print "********************"

    print "Attempting to Login"

    print "********************"

    #ungzipResponse(response, br)

    return br

def ungzipResponseBr(r,b):

    headers = r.info()

    if headers['Content-Encoding']=='gzip':

        import gzip

        print "********************"

        print "Deflating gzip response"

        print "********************"

        gz = gzip.GzipFile(fileobj=r, mode='rb')

        html = gz.read()

        gz.close()

        headers["Content-type"] = "text/html; charset=utf-8"

        r.set_data( html )

        b.set_response(r)