Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from bs4 import BeautifulSoup

from bson.binary import Binary

from datetime import datetime, date, timedelta

from dtr import main

from dtr.api.Service import Orders

from dtr.dao import AffiliateInfo, Order, SubOrder

from dtr.main import getBrowserObject, ScrapeException, getStore, ParseException, \

    Store as MStore, ungzipResponse, tprint

from dtr.storage import Mongo

from dtr.storage.DataService import Order_Parse_Info, All_user_addresses, \

    OrdersRaw

from dtr.storage.Mongo import getImgSrc, getDealRank

from dtr.utils import utils

from dtr.utils.utils import fetchResponseUsingProxy, readSSh

from elixir import *

from pprint import pprint

from pymongo import MongoClient

from pyquery import PyQuery

from urlparse import urlparse, parse_qs

from xlrd import open_workbook

import csv

import json

import os.path

import pymongo

import re

import time

import traceback

import urllib

import urllib2

USERNAME='profittill2@gmail.com'

PASSWORD='spice@2020'

AFFILIATE_URL='http://affiliate.snapdeal.com/login/'

POST_URL='https://api-p03.hasoffers.com/v3/Affiliate_Report.json'

ORDER_TRACK_URL='https://m.snapdeal.com/orderSummary'

CONFIG_URL='http://affiliate.snapdeal.com/publisher/js/config.php'

#NEW_REPORT_URI_TEMPLATE = "http://affiliate.snapdeal.com/affiliate/reports/orders/report/?fromDate=%s&toDate=%s&dump_report=True&request_type=product&status=%s"

AFF_URL_TEMPLATE = "http://affiliate-feeds.snapdeal.com/feed/api/order?startDate=%s&endDate=%s&status=%s"

#"http://affiliate.snapdeal.com/affiliate/reports/orders/report/?fromDate=2015-04-01&toDate=2015-09-15&dump_report=True&request_type=product&status=cancelled

#"http://affiliate.snapdeal.com/affiliate/reports/orders/report/?fromDate=07-09-2015&toDate=13-09-2015&dump_report=True&request_type=product&status=cancelled

AFF_ID = "33550"

AFF_TOKEN="66d526141b9d39c4b2b4ff76eadc34"

headers = { 

            'User-agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',

            'Connection':'keep-alive',

            'Accept-Encoding' : 'gzip,deflate,sdch'

        }

class Store(MStore):

    '''

    This is to map order statuses of our system to order statuses of snapdeal.

    And our statuses will change accordingly.

    '''

    OrderStatusMap = {

                      MStore.ORDER_PLACED : ['in progress', 'pending for verification', 'not available', 'in process', 

                                             'processing', 'processed', 'under verification', 'readying for dispatch',

                                             'waiting for courier to pick up', 'processing initiated','prepared for dispatch','dispatching soon',                                        

                                             'item packed. dispatching soon.','cancellation requested'],

                      MStore.ORDER_DELIVERED : ['delivered','delivered successfully!'],

                      MStore.ORDER_SHIPPED : ['in transit', 'dispatched', 'handed over to courier', 'undelivered. update delivery details!','out for delivery',

                                              'undelivered. edit delivery details!','undelivered' ,'all delivery attempts failed','delivery attempt failed',

                                              'delivery details updated'],

                      MStore.ORDER_CANCELLED : ['closed for vendor reallocation', 'cancelled', 'product returned by courier', 'returned', 'n/a', 'courier returned','verification failed' 

                                                'a new order placed with a different seller', 'closed', 'cancellation in progress', 'verification failed. order cancelled', 

                                                'cancelled. payment refunded','closed. new order placed','cancelling','cancelling order','order cancelled',

                                                'payment failed. order cancelled','returned to snapdeal','pickup sent','refund approved','refund successful!',

                                                'undelivered. order cancelled','order cancelled. refunded successfully','return request registered','pickup failed. please reschedule',

                                                'replacement order successfully placed','replacement request received', 'verification failed due to nbp. order cancelled', 

                                                'verification failed due to where customer don\'t want order. order cancelled']

                      }

    CONF_CB_AMOUNT = MStore.CONF_CB_DISCOUNTED_PRICE

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def getName(self):

        return "snapdeal"

    def scrapeAffiliate(self, startDate=None, endDate=None):

        endDate=date.today() - timedelta(days=1)

        if startDate is None:

            startDate = endDate - timedelta(days=45)

        endDate = endDate.strftime('%Y-%m-%d')

        startDate = startDate.strftime('%Y-%m-%d')

        statuses=['cancelled', 'approved']

        for status in statuses:

            nextUrl = AFF_URL_TEMPLATE%(startDate, endDate, status)

            while nextUrl:

                req = urllib2.Request(nextUrl)

                nextUrl=''

                req.add_header('Snapdeal-Affiliate-Id', AFF_ID)

                req.add_header('Snapdeal-Token-Id', AFF_TOKEN)

                req.add_header('Accept', 'application/json')

                resp = urllib2.urlopen(req)

                ordersDict = json.loads(resp.read())

                self._saveToAffiliate(ordersDict['productDetails'], status)

    def _setLastSaleDate(self, saleDate):

        self.db.lastSaleDtate.update({'storeId':self.store_id}, {'$set':{'saleDate':saleDate}})

    def _getLastSaleDate(self,):

        lastDaySaleObj = self.db.lastDaySale.find_one({"storeId":self.store_id})

        if lastDaySaleObj is None:

            return datetime.min

    def _parseB(self, orderId, subTagId, userId, page, orderSuccessUrl):

        soup = BeautifulSoup(page)

        orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)

        orderDetailDiv = orderDetailContainerDivs.pop(0)

        paymentDetailDiv  = orderDetailContainerDivs.pop(0)

        subOrders = orderDetailContainerDivs

        placedOn = orderDetailDiv.span.text.split(':')[1].strip()

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        merchantOrder.placedOn = placedOn

        merchantOrder.merchantOrderId = parse_qs(urlparse(orderSuccessUrl).query)['order'][0]

        paymentDivs = paymentDetailDiv.findAll('div', recursive=False)

        paymentDivs.pop(0)

        for orderTr in paymentDivs:

            orderTrString = str(orderTr)

            if "Total Amount Paid" in orderTrString:

                amountPaid = orderTr.div.find('div', {'class':'detailBlock'}).text.strip()

                merchantOrder.paidAmount = int(re.findall(r'\d+', amountPaid)[0])

            elif "Total Amount" in orderTrString:

                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]

            elif "Delivery Charges" in orderTrString:

                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]

            elif "Discount Applied" in orderTrString:

                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

            elif "Offer Discount" in orderTrString:

                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

        merchantSubOrders = []

        for subOrderElement in subOrders:

            subOrder = self.parseSubOrderB(subOrderElement, placedOn)

            if subOrder is not None:

                dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

                subOrder.dealRank = dealRank.get('rank')

                subOrder.rankDesc = dealRank.get('description')

                subOrder.maxNlc = dealRank.get('maxNlc')

                subOrder.minNlc = dealRank.get('minNlc')

                subOrder.db = dealRank.get('dp')

                subOrder.itemStatus = dealRank.get('status')

                merchantSubOrders.append(subOrder)

        merchantOrder.subOrders = merchantSubOrders

        return merchantOrder

    def _parse(self, orderId, subTagId, userId, page, orderSuccessUrl):

        #page=page.decode("utf-8")

        soup = BeautifulSoup(page)

        #orderHead = soup.find(name, attrs, recursive, text)

        sections = soup.findAll("section")

        #print sections

        order = sections[1]

        orderTrs = order.findAll("tr")

        placedOn = str(orderTrs[0].findAll("td")[1].text)

        #Pop two section elements

        sections.pop(0) 

        sections.pop(0)

        subOrders = sections

        merchantSubOrders = []

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        merchantOrder.placedOn = placedOn

        merchantOrder.merchantOrderId = re.findall(r'\d+', str(soup.find("div", {"class":"deals_heading"})))[1]

        for orderTr in orderTrs:

            orderTrString = str(orderTr)

            if "Total Amount" in orderTrString:

                merchantOrder.totalAmount = re.findall(r'\d+', orderTrString)[0]

            elif "Delivery Charges" in orderTrString:

                merchantOrder.deliveryCharges = re.findall(r'\d+', orderTrString)[0]

            elif "Discount Applied" in orderTrString:

                merchantOrder.discountApplied = re.findall(r'\d+', orderTrString)[0]

            elif "Paid Amount" in orderTrString:

                merchantOrder.paidAmount = re.findall(r'\d+', orderTrString)[0]

        for subOrderElement in subOrders:

            subOrders = self.parseSubOrder(subOrderElement, placedOn)                           

            merchantSubOrders.extend(subOrders)   

        merchantOrder.subOrders = merchantSubOrders

        return merchantOrder

    def parseSubOrder(self, subOrderElement, placedOn):

        subOrders = []

        productUrl = str(subOrderElement.find("a")['href'])

        subTable = subOrderElement.find("table", {"class":"lrPad"})

        subTrs = subTable.findAll("tr")

        unitPrice=None

        offerDiscount = 0

        deliveryCharges = None

        amountPaid = None

        amount = 0

        sdCash = 0

        unitPrice = 0

        for subTr in subTrs:

            subTrString = str(subTr)

            if "Unit Price" in subTrString:

                unitPrice = int(re.findall(r'\d+', subTrString)[0])

            if "Quantity" in subTrString:

                qty = int(re.findall(r'\d+', subTrString)[0])

            elif "Offer Discount" in subTrString:

                offerDiscount +=   int(re.findall(r'\d+', subTrString)[0])

            elif "SD Cash" in subTrString:

                sdCash =   int(re.findall(r'\d+', subTrString)[0])

            elif "Delivery Charges" in subTrString:

                deliveryCharges =   int(re.findall(r'\d+', subTrString)[0])

            elif "Subtotal" in subTrString:

                if int(qty) > 0:

                    amountPaid =   int(re.findall(r'\d+', subTrString)[0])/qty

                else:

                    amountPaid =   0

        if qty>0:

            amount = unitPrice - offerDiscount - sdCash

            amount = 0 if amount < 0 else amount

        div1 = subOrderElement.find("div", {"class": "blk lrPad subordrs"})

        if div1 is None:

            raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

        for strDiv in str(div1).split("<div class=\"seperator\"></div>"):

            div = BeautifulSoup(strDiv)

            productTitle = str(subOrderElement.find("a").text)

            productUrl = "http://m.snapdeal.com/" + productUrl 

            subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

            subOrder.amountPaid = amountPaid

            subOrder.deliveryCharges = deliveryCharges

            subOrder.offerDiscount = offerDiscount

            subOrder.unitPrice = int(unitPrice)

            subOrder.productCode = re.findall(r'\d+$', productUrl)[0]

            subOrder.imgUrl = Mongo.getImgSrc(subOrder.productCode, self.store_id).get('thumbnail')

            cashbackStatus = Store.CB_NA

            cashbackAmount = 0

            percentage = 0

            if amount > 0:

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)

                if cashbackAmount > 0:

                    cashbackStatus = Store.CB_PENDING

            subOrder.cashBackStatus = cashbackStatus

            subOrder.cashBackAmount = cashbackAmount

            subOrder.cashBackPercentage = percentage

            trackAnchor = div.find("a")   

            if trackAnchor is not None:

                subOrder.tracingkUrl = str(trackAnchor['href'])

            divStr = str(div)

            divStr = divStr.replace("\n","").replace("\t", "")

            for line in divStr.split("<br />"):

                if "Suborder ID" in line:

                    subOrder.merchantSubOrderId = re.findall(r'\d+', line)[0]   

                elif "Status" in line:

                    print line

                    subOrder.detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                elif "Est. Shipping Date" in line:

                    subOrder.estimatedShippingDate = line.split(":")[1].strip()

                elif "Est. Delivery Date" in line:

                    subOrder.estimatedDeliveryDate = line.split(":")[1].strip()

                elif "Courier Name" in line:

                    subOrder.courierName = line.split(":")[1].strip()

                elif "Tracking No" in line:

                    subOrder.trackingNumber = line.split(":")[1].strip()

            subOrders.append(subOrder)

        return subOrders

    def parseSubOrderB(self, subOrderElement, placedOn):

        subOrders = []

        prodDivs = subOrderElement.findAll('div', recursive=False)

        prodDetailDiv = prodDivs[1].findAll('div', recursive=False)

        offerDiscount = 0

        deliveryCharges = None

        amountPaid = 0

        sdCash = 0

        unitPrice = 0

        paymentDivs = prodDivs[2].findAll('div', recursive=False)

        for paymentDiv in paymentDivs:

            strPaymentDiv = str(paymentDiv)

            if "Unit Price" in strPaymentDiv:

                try:

                    unitPrice = int(re.findall(r'\d+', strPaymentDiv)[0])

                except:

                    return None

            elif "Offer Discount" in strPaymentDiv:

                offerDiscount += int(re.findall(r'\d+', strPaymentDiv)[0])

            elif "Discount" in strPaymentDiv:

                offerDiscount += int(re.findall(r'\d+', strPaymentDiv)[0])

            elif "SD Cash" in strPaymentDiv:

                sdCash =   int(re.findall(r'\d+', strPaymentDiv)[0])

            elif "Delivery Charges" in strPaymentDiv:

                deliveryCharges =   int(re.findall(r'\d+', strPaymentDiv)[0])

            elif "Subtotal" in strPaymentDiv:

                amountPaid =   int(re.findall(r'\d+', paymentDiv.find('div', {'class':'itemPriceDetail'}).text)[0])

        amount = unitPrice - offerDiscount - sdCash

        imgDiv = prodDetailDiv[0]

        otherDiv = prodDetailDiv[1]

        productTitle = otherDiv.find('div',{'class':'orderName'}).text.strip()

        productUrl = imgDiv.a['href']

        subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

        subOrder.merchantSubOrderId = prodDivs[0].text.split(':')[1].strip()

        subOrder.detailedStatus = otherDiv.find('div',{'class':'orderStatus'}).span.text.strip()

        if subOrder.detailedStatus.lower() == "processing":

            processingDetailedStatus = subOrderElement.find('div', {'class':'trackingMessage'}).text.strip()

            if processingDetailedStatus.lower() == 'a new order placed with a different seller':

                subOrder.detailedStatus = processingDetailedStatus

        deliveryStatus = otherDiv.find('div',{'class':'orderDelivery'})

        if deliveryStatus is not None:

            delString = deliveryStatus.text.strip()

            arr = delString.split(':')

            if "On" in arr[0]:

                subOrder.deliveredOn = arr[1].strip()

            elif "Exp. Delivery by" in arr[0]:

                subOrder.estimatedDeliveryDate = arr[1].strip()

            elif "Est. delivery between" in arr[0]:

                subOrder.estimatedDeliveryDate = arr[0].split("between")[1].strip()

            elif "Est. shipping between" in arr[0]:

                subOrder.estimatedShippingDate = arr[0].split("between")[1].strip()

            else:

                subOrder.estimatedShippingDate = arr[1].strip()

        subOrder.imgUrl = imgDiv.a.img['src']

        subOrder.productCode = re.findall(r'\d+$', productUrl)[0]

        subOrder.deliveryCharges = deliveryCharges

        subOrder.offerDiscount = offerDiscount

        subOrder.unitPrice = int(unitPrice)

        cashbackStatus = Store.CB_NA

        cashbackAmount = 0

        percentage = 0

        if amountPaid > 0:

            (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amount)

            if cashbackAmount > 0:

                cashbackStatus = Store.CB_PENDING

        subOrder.cashBackStatus = cashbackStatus

        subOrder.cashBackAmount = cashbackAmount

        subOrder.cashBackPercentage = percentage

        courierDet = subOrderElement.find('div', {'class':'courierDetail'})

        if courierDet is not None:

            subOrder.courierName = courierDet.span.text.strip()

        trackingDet = subOrderElement.find('div', {'class':'trackingNo'})

        if trackingDet is not None:

            subOrder.trackingUrl = trackingDet.span.a['href']

            subOrder.trackingNumber = trackingDet.span.a.text.strip()

        subOrders.append(subOrder)

        return subOrder

    def getOrderJSON(self, pq, supcMap):

        #print rawHtml

#        replace_with = {

#        '<': '&gt;',

#        '>': '<',

#        '&': '&',

#        '"': '"', # should be escaped in attributes

#        "'": '''    # should be escaped in attributes

#        }

        jsonValue = pq("#orderJSON").attr("value")

        jsonValue.replace(""", '"')

        jsonValue.replace("&", '&')

        jsonValue.replace(">", '>')

        jsonValue.replace("&lt;", '<')

        jsonValue.replace("'", "'")

        allSupcElements = pq('div.mdt-layout')('div.mdt-card')('div.order-item')

        for supcElement in allSupcElements:

            try:

                supcElement = pq(supcElement)

                title = supcElement('div.order-heading').text().strip()

                productUrl = supcElement.attr("data-href")

                imgUrl = supcElement.find('img').attr('src')

                try:

                    supc = self.catalogdb.MasterData.find_one({"secondaryIdentifier":productUrl.split("/")[-1], "source_id":self.store_id})

                    if supc is None:

                        raise

                    else:

                        supc = supc['identifier'] 

                except:

                    try:

                        supc = imgUrl.split('-')[-3]

                    except:

                        supc = self.db.sdIdentifiers.find_one({"secondaryIdentifier":productUrl.split("/")[-1]})

                        if supc is None:

                            raise

                        else:

                            supc = supc['identifier']

                supcMap[supc] = {'title':title, 'imgUrl':imgUrl, 'productUrl':productUrl}

            except:

                pass

        return json.loads(jsonValue)

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

                    #print merchantOrder

        resp = {}

        orderPart = re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

        url = ORDER_TRACK_URL + orderPart

        moId =  orderPart.split("order=")[-1].split("&")[0]

        if self.db.merchantOrder.find_one({"merchantOrderId":moId}):

            resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp

        supcMap = {}

        try:

            pq = PyQuery(rawHtml)

            try:

                if pq("title").text()=="Webpage not available":

                    raise                

                orderJSON = self.getOrderJSON(pq, supcMap)

            except:

                resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                return resp

                '''page =fetchResponseUsingProxy(url)

                try:

                    merchantOrder = self._parseB(orderId, subTagId, userId, page, orderSuccessUrl)

                except:

                    traceback.print_exc()

                    merchantOrder = self._parse(orderId, subTagId, userId, page, orderSuccessUrl)'''

            merchantOrder = self._parseC(orderId, subTagId, userId, supcMap, orderJSON, orderSuccessUrl)

            merchantOrder.orderTrackingUrl = url

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            print "=================", resp, orderId, "=============="

            return resp

        except:

            print "Error occurred"

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            print "=================", resp, orderId, "=============="

            return resp

        #soup = BeautifulSoup(rawHtml,convertEntities=BeautifulSoup.HTML_ENTITIES)

        #soup.find(name, attrs, recursive, text)

    def _parseC(self, orderId, subTagId, userId, supcMap, orderJSON, orderSuccessUrl):

        print orderJSON

        merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

        placedOn = datetime.strftime(utils.fromTimeStamp(orderJSON['created']/1000), "%a, %d %b, %Y")

        merchantOrder.placedOn = placedOn

        merchantOrder.merchantOrderId = orderJSON['code']

        merchantOrder.paidAmount = orderJSON['paidAmount']

        merchantOrder.deliveryCharges = orderJSON['shippingCharges']

        merchantOrder.closed= False

        merchantSubOrders = []

        for s in orderJSON['suborders']:

            print s

            if not supcMap.has_key(s['supcCode']):

                skuData = Mongo.get_mongo_connection().Catalog.MasterData.find_one({'identifier':s['supcCode'], 'source_id':self.store_id})

                if skuData is None:

                    url = "http://www.snapdeal.com/search?keyword=%s"%s['supcCode']

                    html = utils.fetchResponseUsingProxy(url)

                    html = html.replace("
", "")

                    pq = PyQuery(html)

                    tag = pq('div.product-tuple-listing')

                    for resultDiv in tag:

                        pq_resultDiv = pq(resultDiv)

                        if (pq_resultDiv.attr['data-defaultsupcforfmcg']).strip() == s['supcCode'].strip():

                            productUrl =  pq_resultDiv('div.product-tuple-image').children('a').attr('href')

                            imgUrl =  pq_resultDiv('div.product-tuple-image').children('a').children('img').attr('src')

                            title = pq_resultDiv('p.product-title').text()

                            break

                else:

                    title = skuData['product_name']

                    productUrl = skuData['marketPlaceUrl']

                    imgUrl = skuData['thumbnail']

                supcMap[s['supcCode']] = {'title':title, 'imgUrl':imgUrl, 'productUrl':productUrl}

            map1 = supcMap[s['supcCode']]

            amountPaid = s['paidAmount']

            productTitle = map1['title']

            productUrl = map1['productUrl'] 

            subOrder = SubOrder(productTitle, productUrl, placedOn, amountPaid)

            if(s.get('deliveryDate') is not None):

                print "Delivered On", 

                subOrder.deliveredOn = datetime.strftime(utils.fromTimeStamp(s.get('deliveryDate')/1000),'%d %b, %Y')

                subOrder.status = MStore.ORDER_DELIVERED

                subOrder.detailedStatus = MStore.ORDER_DELIVERED

            elif s['suborderStatus'].get('macroDescription')== 'Closed':

                if s['suborderStatus'].get('value')== 'Close for vendor reallocation':

                    subOrder.detailedStatus = 'Close for vendor reallocation'

                    subOrder.status = MStore.ORDER_CANCELLED

            try:

                subOrder.detailedStatus = s['suborderStatus']['macroDescription']

                subOrder.status = self._getStatusFromDetailedStatus(subOrder.detailedStatus)

            except:

                print "----------------", s['suborderStatus']

            subOrder.merchantSubOrderId = s['code']

            subOrder.deliveryCharges = s['shippingCharges']

            subOrder.productCode = re.findall(r'\d+$', productUrl)[0]

            dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

            subOrder.dealRank = dealRank.get('rank')

            subOrder.rankDesc = dealRank.get('description')

            subOrder.maxNlc = dealRank.get('maxNlc')

            subOrder.minNlc = dealRank.get('minNlc')

            subOrder.db = dealRank.get('dp')

            subOrder.itemStatus = dealRank.get('status')

            subOrder.imgUrl = map1['imgUrl']

            subOrder.unitPrice = s['offerPrice']  -s['internalCashbackValue'] - s['externalCashbackValue']

            subOrder.amount = subOrder.unitPrice - s['offerDiscount'] - s['sdCash']

            try:

                try:

                    if s['shipDateRange']['start']==s['shipDateRange']['end']:

                        subOrder.estimatedShippingDate = datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['start']/1000),'%d %b, %Y') 

                    else:

                        subOrder.estimatedShippingDate = datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['start']/1000),'%d %b, %Y') + " - " + datetime.strftime(utils.fromTimeStamp(s['shipDateRange']['end']/1000),'%d %b, %Y')

                except:   

                    if s['deliveryDateRange']['start']==s['deliveryDateRange']['end']:

                        subOrder.estimatedDeliveryDate = datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['start']/1000),'%d %b, %Y') 

                    else:

                        subOrder.estimatedDeliveryDate = datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['start']/1000),'%d %b, %Y') + " - " + datetime.strftime(utils.fromTimeStamp(s['deliveryDateRange']['end']/1000),'%d %b, %Y')   

            except:

                pass

            subOrder.offerDiscount = s['offerDiscount']

            subOrder.unitPrice = s['offerPrice']

            merchantSubOrders.append(subOrder)

        merchantOrder.subOrders = merchantSubOrders

        self.populateDerivedFields(merchantOrder)

        return merchantOrder

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

            elif 'order cancelled' in detailedStatus.lower():

                return key

        print "Detailed Status need to be mapped", detailedStatus, self.store_id

        return None

    def scrapeStoreOrders(self,):

        #collectionMap = {'palcedOn':1}

        orders = self._getActiveOrders()

        for order in orders:

            order = obj(order)

            print "Order", self.store_name, order.orderId

            try:

                url = order.orderTrackingUrl

                page = fetchResponseUsingProxy(url, headers=headers)

                #page=page.decode("utf-8")

                try:

                    pq = PyQuery(page)

                    subOrderStatusMap={}

                    for el in pq('div.cardLayout.pad-10.mb-10'):

                        elpq = PyQuery(el)

                        try:

                            subOrderId = elpq("div.subOrderId").text().split(":")[1].strip()

                            subOrderStatusMap[subOrderId] = elpq

                        except:

                            pass

                    closedForReco = {}

                    for suborder in order.subOrders:

                        if suborder.closed:

                            if suborder.merchantSubOrderId in subOrderStatusMap:

                                del subOrderStatusMap[suborder.merchantSubOrderId]

                                continue

                        if subOrderStatusMap.has_key(suborder.merchantSubOrderId):

                            elpq = subOrderStatusMap.get(suborder.merchantSubOrderId)

                            del subOrderStatusMap[suborder.merchantSubOrderId]

                            if elpq("#trackLink").attr("href"):

                                suborder.trackingUrl = elpq("#trackLink").attr("href")

                            if elpq('span.subOrdStatusText').text():

                                suborder.estimatedDeliveryDate=elpq('span.subOrdStatusText').text().strip()

                            suborder.detailedStatus = elpq("div.orderStatus span").text().strip()

                            if suborder.detailedStatus in ['Closed. New Order Placed','Closed. Placing New Order'] :

                                closedForReco[suborder.merchantSubOrderId] = suborder

                            suborder.status = self._getStatusFromDetailedStatus(suborder.detailedStatus)

                    if len(closedForReco) ==  len(subOrderStatusMap) and len(closedForReco)>0:

                        productCode = ''

                        allProductsSame = True

                        for subOrderId, subo in closedForReco.iteritems():

                            if productCode == '':

                                productCode = subo.productCode

                                continue

                            if subo.productCode != productCode:

                                allProductsSame = False

                                break

                        subOrderStatusMap

                        if allProductsSame:

                            print "singlereco", order.orderId

                            for key, elpq in subOrderStatusMap.iteritems():

                                suborderNew = obj(todict(suborder))

                                suborderNew.merchantSubOrderId = key

                                if elpq("#trackLink").attr("href"):

                                    suborderNew.trackingUrl = elpq("#trackLink").attr("href")

                                if elpq('span.subOrdStatusText').text():

                                    suborderNew.estimatedDeliveryDate=elpq('span.subOrdStatusText').text().strip()

                                suborderNew.detailedStatus = elpq("div.orderStatus span").text().strip()

                                suborderNew.status = self._getStatusFromDetailedStatus(suborderNew.detailedStatus)

                                order.subOrders.append(suborderNew)

                        else:

                            print "All products not same referring transaction url", order.orderId

                    else: 

                        pass

                        #Lookout for ordersummary page for exact mapping        

                    self.populateDerivedFields(order, update=True)

                    self._updateToOrder(todict(order))

                except:

                    traceback.print_exc()

                    soup = BeautifulSoup(page)

                    try:

                        self.tryBParsing(order, soup)

                    except:

                        traceback.print_exc()

                        sections = soup.findAll("section")

                        orderEl = sections[1]

                        orderTrs = orderEl.findAll("tr")

                        placedOn = str(orderTrs[0].findAll("td")[1].text)

                        sections.pop(0)

                        sections.pop(0)

                        subOrders = sections

                        bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                        closed = True

                        for subOrderElement in subOrders:

                            div1 = subOrderElement.findAll("div", {"class": "blk lrPad subordrs"})

                            if len(div1)<=0:

                                raise ParseException("subOrder", "Could not Parse suborders for Snapdeal")

                            subOrder = None

                            breakFlag = False

                            for strDiv in str(div1).split("<div class=\"seperator\"></div>"):

                                div = BeautifulSoup(strDiv)

                                divStr = str(div)

                                divStr = divStr.replace("\n","").replace("\t", "")

                                updateMap = {}

                                for line in divStr.split("<br />"):

                                    if "Suborder ID" in line:

                                        merchantSubOrderId = re.findall(r'\d+', line)[0]

                                        #break if suborder is inactive   

                                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

                                        if subOrder is None:

                                            subOrders = self.parseSubOrder(subOrderElement, placedOn)

                                            self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict(subOrders)}}})

                                            print "Added new suborders to Order id - ", order['orderId']

                                            closed = False

                                            breakFlag = True

                                            break

                                        elif subOrder['closed']:

                                            breakFlag = True

                                            break

                                        else: 

                                            findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                                    elif "Status :" in line:

                                        detailedStatus = re.findall('>(.*?)</span>', line, re.IGNORECASE)[0]

                                        updateMap["subOrders.$.detailedStatus"] = detailedStatus

                                        status = self._getStatusFromDetailedStatus(detailedStatus) 

                                        closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                                        if status is not None:

                                            updateMap["subOrders.$.status"] = status

                                        if detailedStatus == 'Closed For Vendor Reallocation':

                                            #if it is more than 6hours mark closed.

                                            closeAt = subOrder.get("closeAt") 

                                            if closeAt is None:

                                                closeAt = datetime.now() + timedelta(hours=6)

                                                updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")

                                            else:

                                                closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")

                                                if datetime.now() > closeAt:

                                                    closedStatus = True

                                        #Close if not applicable suborders are not closed

                                        if utils.fromTimeStamp(order['createdOnInt'] + 35*86400*1000) < datetime.now() and subOrder['cashBackStatus']==utils.CB_NA:

                                            closedStatus=True   

                                        if closedStatus:

                                            #if status is closed then change the paybackStatus accordingly

                                            updateMap["subOrders.$.closed"] = True

                                            if status == Store.ORDER_DELIVERED:

                                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                                            elif status == Store.ORDER_CANCELLED:

                                                if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                                                    updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                                        else:

                                            closed = False

                                    elif "Est. Shipping Date" in line:

                                        estimatedShippingDate = line.split(":")[1].strip()

                                        updateMap["subOrders.$.estimatedShippingDate"] = estimatedShippingDate

                                    elif "Est. Delivery Date" in line:

                                        estimatedDeliveryDate = line.split(":")[1].strip()

                                        updateMap["subOrders.$.estimatedDeliveryDate"] = estimatedDeliveryDate

                                    elif "Courier Name" in line:

                                        courierName = line.split(":")[1].strip()

                                        updateMap["subOrders.$.courierName"] = courierName

                                    elif "Tracking No" in line:

                                        trackingNumber = line.split(":")[1].strip()

                                        updateMap["subOrders.$.trackingNumber"] = trackingNumber

                                if breakFlag:

                                    continue

                                bulk.find(findMap).update({'$set' : updateMap})

                            bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})

                        result = bulk.execute()

                        tprint(result)

            except:

                traceback.print_exc()                

                tprint("Could not update " + str(order.orderId) + "For store " + self.getName())

                self.db.merchantOrder.update({"orderId":order.orderId}, {"$set":{"parseError":True}})

    def tryBParsing(self, order, soup):

        orderDetailContainerDivs = soup.body.find("div", {'class':'cardLayoutWrap'}).findAll('div', recursive=False)

        orderDetailDiv = orderDetailContainerDivs.pop(0)

        placedOn = orderDetailDiv.span.text.split(':')[1].strip()

        orderDetailContainerDivs.pop(0)

        subOrders = orderDetailContainerDivs

        bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

        closed = True

        for subOrderElement in subOrders:

            prodDivs = subOrderElement.findAll('div', recursive=False)

            merchantSubOrderId = prodDivs[0].text.split(':')[1].strip()

            subOrder = None

            subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

            if subOrder is None:

                try:

                    subOrder = self.parseSubOrderB(subOrderElement, placedOn)

                    if subOrder is None:

                        continue

                    self.db.merchantOrder.update({"orderId":order['orderId']},{'$push':{"subOrders":{"$each":todict([subOrder])}}})

                    print "Added new suborders to Order id - ", order['orderId']

                    closed = False

                except:

                    pass

                continue

            elif subOrder['closed']:

                continue

            else: 

                prodDetailDiv = prodDivs[1].findAll('div', recursive=False)

                otherDiv = prodDetailDiv[1]

                trackBlock = subOrderElement.find('div',{'class':'trackingDetailsBlock'})

                findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                updateMap = {}

                detailedStatus = otherDiv.find('div',{'class':'orderStatus'}).span.text.strip()

                if 'A new order placed with a different seller' in str(trackBlock):

                    #if it is more than 6hours mark closed.

                    closeAt = subOrder.get("closeAt") 

                    if closeAt is None:

                        closeAt = datetime.now() + timedelta(hours=6)

                        updateMap["subOrders.$.closeAt"] = datetime.strftime(closeAt,"%Y-%m-%d %H:%M:%S")

                        bulk.find(findMap).update({'$set' : updateMap})

                        closed=False

                        continue

                    else:

                        closeAt = datetime.strptime(closeAt,"%Y-%m-%d %H:%M:%S")

                        if datetime.now() > closeAt:

                            detailedStatus = 'A new order placed with a different seller'

                status = self._getStatusFromDetailedStatus(detailedStatus) 

                closedStatus = status in [Store.ORDER_DELIVERED, Store.ORDER_CANCELLED]

                updateMap["subOrders.$.detailedStatus"] = detailedStatus

                if status is not None:

                    updateMap["subOrders.$.status"] = status

                if closedStatus:

                    #if status is closed then change the paybackStatus accordingly

                    updateMap["subOrders.$.closed"] = True

                    if status == Store.ORDER_DELIVERED:

                        if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                            updateMap["subOrders.$.cashBackStatus"] = Store.CB_APPROVED

                    elif status == Store.ORDER_CANCELLED:

                        if subOrder.get("cashBackStatus") == Store.CB_PENDING:

                            updateMap["subOrders.$.cashBackStatus"] = Store.CB_CANCELLED

                else:

                    closed = False

                deliveryStatus = otherDiv.find('div',{'class':'orderDelivery'})

                if deliveryStatus is not None:

                    delString = deliveryStatus.text.strip()

                    arr = delString.split(':')    

                    if "On" in arr[0]:

                        updateMap['subOrders.$.deliveredOn'] = arr[1].strip()

                    elif "Exp. Delivery by" in arr[0]:

                        updateMap['subOrders.$.estimatedDeliveryDate'] = arr[1].strip()

                    elif "Est. delivery between" in arr[0]:

                        updateMap['subOrders.$.estimatedDeliveryDate'] = delString.split("between")[1].strip()

                    elif "Est. shipping between" in arr[0]:

                        updateMap['subOrders.$.estimatedShippingDate'] = delString.split("between")[1].strip()

                    else:

                        updateMap['subOrders.$.estimatedShippingDate'] = arr[1].strip()

                courierDet = subOrderElement.find('div', {'class':'courierDetail'})

                if courierDet is not None:

                    updateMap['subOrders.$.courierName'] = courierDet.span.text.strip()

                trackingDet = subOrderElement.find('div', {'class':'trackingNo'})

                if trackingDet is not None:

                    updateMap['subOrders.$.trackingUrl'] = trackingDet.span.a['href']

                    updateMap['subOrders.$.trackingNumber'] = trackingDet.span.a.text.strip()                    

            bulk.find(findMap).update({'$set' : updateMap})

        bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed,"parseError":False}})

        result = bulk.execute()

        tprint(result)

    def _saveToAffiliate(self, offers, status):

        collection = self.db.snapdealOrderAffiliateInfo1

        #mcollection = self.db.merchantOrder

        for offer in offers:

                offer = self.covertToObj(offer)

                if offer.orderId:

                    dict1 = todict(offer)

                    dict1["_id"] = dict1["orderId"] + "-" + dict1["productCode"]

                    dict1['status'] = status

                    collection.save(dict1)

#    def _saveToAffiliate(self, offers):

#        collection = self.db.snapdealOrderAffiliateInfo

#        mcollection = self.db.merchantOrder

#        for offer in offers:

#            offer = self.covertToObj(offer)

#            collection.update({"adId":offer.adId, "saleAmount":offer.saleAmount, "payOut":offer.payOut},{"$set":todict(offer)}, upsert=True)

#            mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True}, {"$set":{"subOrders.$.missingAff":False}})

    def _getAllOffers(self, br, token):

        allOffers = []

        nextPage = 1  

        while True:

            data = getPostData(token, nextPage)

            response = br.open(POST_URL, data)

            rmap = json.loads(ungzipResponse(response))

            if rmap is not None:

                rmap = rmap['response']

                print rmap

                if rmap is not None and len(rmap['errors'])==0:

                    allOffers += rmap['data']['data']

            nextPage += 1

            if rmap['data']['pageCount']<nextPage:

                break

        return allOffers

#    def covertToObj(self,offer):

#        offerData = offer['Stat']

#        offer1 = AffiliateInfo(offerData['affiliate_info1'], self.store_id, offerData['conversion_status'], offerData['ad_id'], 

#                              offerData['datetime'], int(float(offerData['payout'])), offer['Offer']['name'], offerData['ip'], int(float(offerData['conversion_sale_amount'])))

#        offer1.saleTime = int(time.mktime(datetime.strptime(offer1.saleDate, "%Y-%m-%d %H:%M:%S").timetuple()))

#        return offer1

    def parseInfo(self,):

        from pyquery import PyQuery as pq

        orders = list(session.query(Orders).filter_by(store_id=self.store_id).filter_by(status='ORDER_CREATED').group_by(Orders.user_id).all())

        try:

            for order in orders:

                try:

                    doc = pq(order.rawhtml)

                    a1= " ".join(["" if not div.text else div.text.replace("\t","").replace("\n","").strip() for div in pq(doc('article')[-1])('div')]).strip()

                    a2 = ",".join(["" if not div.text else div.text.replace("\t","").replace("\n","").replace(" ", "") for div in pq(doc('article')[-2])('div')]).strip()

                    user_address = All_user_addresses()

                    user_address.address = a1

                    all = a2.split(",")

                    user_address.source = 'order'

                    user_address.user_id = order.user_id

                    #user_address. = all[3].split(":")[1]

                    #user_address. = all[2].split(":")[1]

                    #orderInfo.mobile = all[-1].split(":")[1]

                    adSplit = a1.split(",")

                    user_address.city = adSplit[-2].strip()

                    user_address.pincode = adSplit[-1].strip().split(" ")[0]

                    user_address.state = adSplit[-1].strip().split(" ")[1]

                    session.commit()

                except:

                    session.rollback()

                    continue

        finally:

            session.close()

    def covertToObj(self,offer):

        offer1 = AffiliateInfo(offer["affiliateSubId1"], 3, None, None, utils.toTimeStamp(datetime.strptime(offer["dateTime"], "%m/%d/%Y %H:%M:%S")),   

                               offer["commissionEarned"], None, None, offer["sale"])

        offer1.orderId = offer.get('orderCode') if offer.get('orderCode')  else None

        offer1.productCode = offer["product"]

        offer1.unitPrice = offer["price"]

        offer1.quantity = offer["quantity"]

        offer1.saleTime = offer["dateTime"] 

        return offer1

def getPostData(token, page = 1, limit= 20, startDate=None, endDate=None):

    endDate=date.today() + timedelta(days=1)

    startDate=endDate - timedelta(days=31)

    parameters = (

        ("page",str(page)),

        ("limit",str(limit)),

        ("fields[]","Stat.offer_id"),

        ("fields[]","Stat.datetime"),

        ("fields[]","Offer.name"),

        ("fields[]","Stat.conversion_status"),

        ("fields[]","Stat.conversion_sale_amount"),

        ("fields[]","Stat.payout"),

        ("fields[]","Stat.ip"),

        ("fields[]","Stat.ad_id"),

        ("fields[]","Stat.affiliate_info1"),

        ("sort[Stat.datetime]","desc"),

        ("filters[Stat.date][conditional]","BETWEEN"),

        ("filters[Stat.date][values][]",startDate.strftime('%Y-%m-%d')),

        ("filters[Stat.date][values][]",endDate.strftime('%Y-%m-%d')),

        ("data_start",startDate.strftime('%Y-%m-%d')),

        ("data_end",endDate.strftime('%Y-%m-%d')),

        ("Method","getConversions"),

        ("NetworkId","jasper"),

        ("SessionToken",token),

    )

    #Encode the parameters

    return urllib.urlencode(parameters)

def main():

    #print todict([1,2,"3"])

    store = getStore(3)

    store.scrapeStoreOrders()

    #store.parseOrderRawHtml(332222, "3232311", 2, readSSh("/home/amit/sample1.html"), "    https://m.snapdeal.com/purchaseMobileComplete?code=a8b3420d4c5bc248ea887df6c9b3a724&order=10478372453")

    #store.scrapeAffiliate()

    #https://m.snapdeal.com/purchaseMobileComplete?code=3fbc8a02a1c4d3c4e906f46886de0464&order=5808451506

    #https://m.snapdeal.com/purchaseMobileComplete?code=9f4dfa49ff08a16d04c5e4bf519506fc&order=9611672826

#    orders = list(session.query(OrdersRaw).filter_by(store_id=3).filter_by(status='ORDER_NOT_CREATED').all())

#    for o in orders:

#        result = store.parseOrderRawHtml(o.id, o.sub_tag, o.user_id, o.rawhtml, o.order_url)['result']

#        o.status = result