Subversion Repositories SmartDukaan

# coding=utf-8

'''

Created on Jan 15, 2015

@author: amit

'''

from base64 import encode

from bs4 import BeautifulSoup

from datetime import datetime, timedelta

from dtr.dao import Order, SubOrder, AmazonAffiliateInfo

from dtr.main import getStore, Store as MStore, ParseException, getBrowserObject, \

    ungzipResponse, tprint

from dtr.sources.flipkart import todict, AFF_REPORT_URL

from dtr.utils.utils import fetchResponseUsingProxy

from paramiko import sftp

from paramiko.client import SSHClient

from paramiko.sftp_client import SFTPClient

import base64

import gzip

import mechanize

import os.path

import paramiko

import re

import time

import traceback

import urllib2

from dtr.storage.Mongo import getDealRank

from dtr.storage.DataService import OrdersRaw

from elixir import *

ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'

ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'

THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'

AMAZON_AFF_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/listReports'

AMAZON_AFF_FILE_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/getReport?filename=saholic-21-orders-report-%s.tsv.gz'

class Store(MStore):

    orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],

                            MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],

                            MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],

                            MStore.ORDER_DELIVERED : ['delivered']

                           }

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def getName(self):

        return "amazon"

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl, track=False):

        parseString = "Tracking" if track else "Transacted"

        print parseString, "Order Id to be parsed is :", orderId

        resp = {}

        resp['result'] = 'ORDER_NOT_CREATED'

        if ORDER_SUCCESS_URL in orderSuccessUrl or THANKYOU_URL in orderSuccessUrl:

            try:

                soup = BeautifulSoup(rawHtml)

                try:

                    orderUrl = soup.find('div', {"id":"thank-you-box-wrapper"}).div.findAll('div', recursive=False)[1].a['href']

                    merchantOrderId = re.findall(r'.*&oid=(.*)&?.*?', orderUrl)[0]

                except:

                    merchantOrderId = soup.find(id="orders-list").div.span.b.text

                order = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl, True)

                order.orderTrackingUrl = ORDER_REDIRECT_URL % (merchantOrderId)

                order.orderSuccessUrl = orderSuccessUrl

                order.merchantOrderId = merchantOrderId

                order.requireDetail = True

                order.status = 'html_required'

                order.closed = None

                if self._saveToOrder(todict(order)):

                    resp['result'] = 'ORDER_CREATED'

                    resp["url"] = ORDER_REDIRECT_URL % (merchantOrderId)

                    resp["htmlRequired"] = True

                    resp['orderId'] = orderId

                else:

                    resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            except:

                #Write all cases here for Order Not created Known

                try:

                    if 'Securely redirecting you' in soup.h3.text.strip():

                        resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                    else:

                        raise

                except:

                    try:

                        if soup.h1.text.strip() in ['This is a duplicate order', 'There was a problem with your payment.']:

                            resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                        else:

                            raise

                    except:

                        try:

                            if soup.h2.text.strip() == 'Web page not available':

                                resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                            else:

                                raise

                        except:

                            resp['result'] = 'ORDER_NOT_CREATED_UNKNOWN'

        else:

            try:

                mo = self.db.merchantOrder.find_one({"orderId":orderId})

                if mo is not None:

                    merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl, False)

                    merchantOrder.createdOn = mo.get("createdOn")

                    merchantOrder.createdOnInt =  mo.get("createdOnInt")

                else:

                    print "Could not find amazon order with order Id", orderId

                    merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

                soup = BeautifulSoup(rawHtml)

                try:

                    self.parseNewStlye(merchantOrder, soup)

                    resp['result'] = 'DETAIL_CREATED'

                except:

                    try:

                        traceback.print_exc()

                        self.parseOldStlye(merchantOrder, soup)

                        resp['result'] = 'DETAIL_CREATED'

                    except:

                        traceback.print_exc()

                        try:

                            self.parseCancelled(merchantOrder, soup)

                            resp['result'] = 'ORDER_CANCELLED'

                        except:

                            try:

                                if soup.h1.span.text=="Account":

                                    resp['result'] = 'DETAIL_NOT_CREATED_KNOWN'

                                else:

                                    raise

                            except:

                                if soup.find(id="ap_signin_pagelet_title").h1.text.strip()=="Sign In":

                                    resp['result'] = 'DETAIL_NOT_CREATED_KNOWN'

                                else:

                                    raise

            except:

                order = self.db.merchantOrder.find_one({"orderId":orderId})

                if order is not None:

                    self.db.merchantOrder.update({"orderId":orderId}, {"$set":{"status":"parse_failed"}})

                print "Error occurred"

                resp['result'] = 'DETAIL_NOT_CREATED_UNKNOWN'

                traceback.print_exc()

        return resp    

    #This should be exposed from api for specific sources

    def scrapeStoreOrders(self):

        orders = self.db.merchantOrder.find({"storeId":1, "closed":False, "subOrders.closed":False, "subOrders.trackingUrl":{"$exists":True}})

        for merchantOrder in orders:

            executeBulk = False

            try:

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                closed = True

                map1 = {}

                for subOrder in merchantOrder.get("subOrders"):

                    if subOrder.get("closed"):

                        continue

                    elif subOrder.get("trackingUrl") is None:

                        closed = False

                        continue

                    executeBulk = True

                    findMap = {"orderId":merchantOrder.get("orderId"), "subOrders.merchantSubOrderId":subOrder.get("merchantSubOrderId")} 

                    trackingUrl = subOrder.get("trackingUrl")

                    if not map1.has_key(trackingUrl):

                        map1[trackingUrl] = self.parseTrackingUrl(trackingUrl)

                    newOrder = map1.get(trackingUrl)

                    newOrder['cashBackStatus'] = subOrder.get('cashBackStatus')

                    updateMap = self.getUpdateMap(newOrder)

                    print findMap, "\n", updateMap

                    bulk.find(findMap).update({'$set' : updateMap})

                    closed = closed and newOrder['closed']

                if executeBulk:

                    bulk.find({"orderId":merchantOrder.get("orderId")}).update({"$set":{"closed":closed, "parseError":False}})

                    bulk.execute()

            except:

                tprint("Could not update " + str(merchantOrder['orderId']) + " For store " + self.getName())

                self.db.merchantOrder.update({"orderId":merchantOrder['orderId']}, {"$set":{"parseError":True}})

                traceback.print_exc()

    def parserest(self, soup):

        print "Hi"

        if soup.find('h1'):

            print "OK"

    def parseOldStlye(self, merchantOrder, soup):

        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl

        table = soup.body.findAll("table", recursive=False)[1]

        #print table

        tables = table.tr.td.findAll("table", recursive=False)

        for tr in tables[2].findAll("tr"):

            boldElement = tr.td.b

            if "Order Placed" in str(boldElement):

                merchantOrder.placedOn = boldElement.next_sibling.strip()

            if "order number" in str(boldElement):

                merchantOrder.merchantOrderId = boldElement.next_sibling.strip()

            if "Order Total" in str(boldElement):

                merchantOrder.paidAmount = int(float(boldElement.find('span').contents[-1].replace(',','')))

        anchors = table.tr.td.findAll("a", recursive=False)

        paymentAnchor = anchors.pop(-1)

        count = 0

        subOrders = []

        merchantOrder.subOrders = subOrders

        counter = 0 

        for anchor in anchors:

            count += 1

            tab = anchor.next_sibling

            status = MStore.ORDER_PLACED

            subStr = "Delivery #" + str(count) + ":"

            if subStr in  tab.find("b").text:

                detailedStatus = tab.find("b").text.replace(subStr, '').strip()

            tab = tab.next_sibling.next_sibling

            trs = tab.find("table").find('tbody').findAll("tr", recursive = False)

            estimatedDelivery = trs[0].td.find("b").next_sibling.strip()

            orderItemTrs = trs[1].findAll("td", recursive=False)[1].table.tbody.findAll("tr", recursive = False)

            i = -1

            for orderItemTr in orderItemTrs:

                i += 1

                if i%2 == 0:

                    continue

                counter += 1

                quantity =  int(re.findall(r'\d+', orderItemTr.td.contents[0])[0])

                productUrl = orderItemTr.td.contents[1].a["href"]

                productTitle = orderItemTr.td.contents[1].a.text

                unitPrice = int(float(orderItemTr.findAll('td')[1].span.text.replace('Rs. ','').replace(',','')))

                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, unitPrice*quantity, status, quantity)

                subOrder.merchantSubOrderId = str(counter) + " of " + merchantOrder.merchantOrderId

                subOrder.estimatedDeliveryDate = estimatedDelivery

                estDlvyTime = datetime.strptime(estimatedDelivery.split('-')[0].strip(), "%A %d %B %Y")

                createdOn = datetime.fromtimestamp(merchantOrder.createdOnInt)

                subOrder.trackAfter = int(time.mktime(max(estDlvyTime-timedelta(days=4),createdOn + timedelta(days=3)).timetuple()))

                subOrder.productCode = productUrl.split('/')[5]

                subOrder.detailedStatus = detailedStatus

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, unitPrice*quantity)

                dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

                subOrder.dealRank = dealRank.get('rank')

                subOrder.rankDesc = dealRank.get('description')

                cashbackStatus = Store.CB_PENDING

                if cashbackAmount <= 0:

                    cashbackStatus = Store.CB_NA

                subOrder.cashBackStatus = cashbackStatus

                subOrder.cashBackAmount = cashbackAmount

                if percentage > 0:

                    subOrder.cashBackPercentage = percentage

                subOrders.append(subOrder)

        priceList = paymentAnchor.next_sibling.next_sibling.next_sibling.table.table.tbody.tbody.tbody.findAll('tr', recursive=False)

        totalAmount = 0

        grandAmount = 0

        for price in priceList:

            labelTd = price.td

            if 'Subtotal:' in labelTd.text:

                totalAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))

            elif 'Grand Total:' in labelTd.text:

                grandAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))

        if grandAmount < totalAmount:

            diff = totalAmount - grandAmount

            for subOrder in merchantOrder.subOrders:

                subOrder.amountPaid -= int(diff*(1-subOrder.amountPaid/totalAmount))

        merchantOrder.status='success'

        self._updateToOrder(todict(merchantOrder))

    def parseNewStlye(self, merchantOrder, soup):

        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl

        orderDetailsContainer = soup.body.find(id="orderDetails")

        divAfterH1 = orderDetailsContainer.h1.next_sibling.next_sibling

        orderLeftDiv = divAfterH1.div

        placedOnSpan = orderLeftDiv.find("span", {'class':'order-date-invoice-item'})

        merchantOrder.placedOn =placedOnSpan.text.split('Ordered on')[1].strip()

        merchantOrder.merchantOrderId = placedOnSpan.next_sibling.next_sibling.text.split('Order#')[1].strip()

        try:

            priceBox = divAfterH1.next_sibling.next_sibling.next_sibling.next_sibling.find("div", {"class":"a-box-inner"}).div.div.findAll('div', recursive=False)[-1]

        except:

            priceBox = divAfterH1.next_sibling.next_sibling.next_sibling.next_sibling.find("div", {"class":"a-box a-last"}).div.div.findAll('div', recursive=False)[-1]

        priceRows = priceBox.findAll('div', {'class':'a-row'})

        subTotal = 0

        shippingPrice = 0   

        promoApplied = 0

        for priceRow in priceRows:

            if "Item(s) Subtotal:" in str(priceRow):

                subTotal = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Shipping:" in str(priceRow):

                shippingPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Grand Total:" in str(priceRow):

                grandPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

                merchantOrder.paidAmount = grandPrice

            elif "Total:" in str(priceRow):

                totalPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Promotion Applied:" in str(priceRow):

                promoApplied  += int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

        totalPaid = subTotal        

        if promoApplied > 0:

            totalPaid -= promoApplied

            if shippingPrice <= promoApplied:

                totalPaid  += shippingPrice

        shipmentDivs = orderDetailsContainer.find('div', class_='shipment').findAll('div', recursive = False)

        subOrders = []

        merchantOrder.subOrders = subOrders

        i=1

        closedStatus = True

        for shipmentDiv in shipmentDivs:

            innerBoxes = shipmentDiv.findAll('div', recursive = False)

            statusDiv = innerBoxes[0]

            subOrderStatus = statusDiv.div.span.text.strip()

            deliverySpan = statusDiv.div.div.find_next_sibling('div').span

            productDivs = innerBoxes[-1].div.div.div.findAll('div', recursive=False)

            subOrders = []

            merchantOrder.subOrders = subOrders

            for i, productDiv in enumerate(productDivs):

                i +=1 

                imgDiv  = productDiv.div.div

                detailDiv = imgDiv.find_next_sibling('div')

                detailDivs = detailDiv.findAll('div', recursive=False)

                arr = detailDivs[0].a.text.strip().split(" of ", 1)

                (productTitle, quantity) = (arr[-1], (1 if len(arr)==1 else int(arr[0])) )

                try:

                    unitPrice = int(float(detailDivs[2].span.text.replace('Rs. ','').replace(',','')))

                except:

                    unitPrice = int(float(detailDivs[3].span.text.replace('Rs. ','').replace(',','')))

                amountPaid = int((unitPrice*quantity*totalPaid)/subTotal)

                productUrl = "http://www.amazon.in" + detailDivs[0].a.get('href')

                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid, MStore.ORDER_PLACED, quantity)

                subOrder.productCode = productUrl.split('/')[5]

                subOrder.unitPrice = unitPrice

                subOrder.merchantSubOrderId = str(i) + " of " + merchantOrder.merchantOrderId

                estDlvyTime = datetime.now()

                if deliverySpan is not None:

                    try:

                        subOrder.estimatedDeliveryDate = deliverySpan.span.text.strip()

                        estDate = subOrder.estimatedDeliveryDate.split("-")[0].strip()

                        subOrder.estimatedDeliveryInt = int(time.mktime((datetime.strptime(estDate, "%A %d %B %Y")).timetuple()))

                        estDlvyTime = datetime.strptime(estDate, "%A %d %B %Y")

                    except:

                        if "Delivered on" in deliverySpan.text:

                            subOrder.deliveredOn = deliverySpan.text.split(":")[1].strip() 

                        subOrder.estimatedDeliveryDate = "Not available"

                createdOn = datetime.fromtimestamp(merchantOrder.createdOnInt)

                subOrder.trackAfter = int(time.mktime(max(estDlvyTime-timedelta(days=4),createdOn + timedelta(days=3)).timetuple()))

                subOrder.detailedStatus = subOrderStatus

                subOrder.deliveryCharges = shippingPrice

                subOrder.imgUrl = imgDiv.img["src"]

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amountPaid)

                dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

                subOrder.dealRank = dealRank.get('rank')

                subOrder.rankDesc = dealRank.get('description')

                cashbackStatus = Store.CB_PENDING

                if cashbackAmount <= 0:

                    cashbackStatus = Store.CB_NA

                subOrder.cashBackStatus = cashbackStatus

                subOrder.cashBackAmount = cashbackAmount

                if percentage > 0:

                    subOrder.cashBackPercentage = percentage

                if hasattr(subOrder, 'deliveredOn'):                               

                    subOrder.status = Store.ORDER_DELIVERED

                    subOrder.closed = True

                    if subOrder.cashBackStatus == Store.CB_PENDING:

                        subOrder.cashBackStatus = Store.CB_APPROVED

                elif closedStatus:

                    closedStatus= False

                subOrders.append(subOrder)

        merchantOrder.status='success'

        merchantOrder.closed = closedStatus

        self._updateToOrder(todict(merchantOrder))

    def parseCancelled(self, merchantOrder,soup):

        try:

            fonts = soup.body.findAll("table", recursive=False)[1].findAll("font")

            if fonts[0].text == "Important Message":

                if fonts[1].text=="This order has been cancelled.":

                    merchantOrder.closed = True

                    merchantOrder.status = "cancelled"

                    merchantOrder.requireDetail = False

                    self._updateToOrder(todict(merchantOrder))

                    return

                else:

                    raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

            else:

                raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

        except:

            orderDetails = soup.body.find(id="orderDetails")

            if orderDetails is not None and orderDetails.h4.text == "This order has been cancelled.":

                merchantOrder.closed = True

                merchantOrder.status = "cancelled"

                merchantOrder.requireDetail = False

                self._updateToOrder(todict(merchantOrder))

            else:

                raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

    def getTrackingUrls(self, userId):

        missingOrderUrls = []

        missingOrders = self._getMissingOrders({'userId':userId})

        for missingOrder in missingOrders:

            missingOrderUrls.append(ORDER_REDIRECT_URL%(missingOrder['merchantOrderId']))

        orders = self._getActiveOrders({'userId':userId, "subOrders.trackingUrl":{"$exists":False} })

        count = len(orders)

        print "count", count

        print "Missing Urls"

        print "*************"

        print missingOrderUrls

        if count > 0:

            return missingOrderUrls + ['https://www.amazon.in/gp/css/order-history', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled&startIndex=10']

        else: 

            return missingOrderUrls

    def trackOrdersForUser(self, userId, url, rawHtml):

        directory = "/AmazonTrack/User" + str(userId)

        if not os.path.exists(directory):

            os.makedirs(directory)

        try:

            searchMap = {'userId':userId}

            collectionMap = {'merchantOrderId':1}

            activeOrders = self._getActiveOrders(searchMap, collectionMap)

            datetimeNow = datetime.now()

            timestamp = int(time.mktime(datetimeNow.timetuple()))

            print "url----------------", url

            if url == 'https://www.amazon.in/gp/css/order-history' or 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled' in url:

                if url == 'https://www.amazon.in/gp/css/order-history':

                    filename = directory + "/orderSummary" + datetime.strftime(datetime.now(), '%d-%m:%H:%M:%S')

                else:   

                    filename = directory + "/cancelledSummary" + datetime.strftime(datetime.now(), '%d-%m:%H:%M:%S')   

                f = open(filename,'w')

                f.write(rawHtml) # python will convert \n to os.linesep

                f.close() # you can omit in most cases as the destructor will call if

                soup = BeautifulSoup(rawHtml)

                allOrders = soup.find(id="ordersContainer").findAll('div', {'class':'a-box-group a-spacing-base order'})

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                for activeOrder in activeOrders:

                    for orderEle in allOrders:

                        orderdiv = orderEle.find('div', {'class':'a-box a-color-offset-background order-info'}).find('div', {'class':'a-fixed-right-grid-col actions a-col-right'})

                        merchantOrderId = orderdiv.find('span', {'class':'a-color-secondary value'}).text.strip()

                        if merchantOrderId==activeOrder['merchantOrderId']:

                            closed = True

                            shipments = orderEle.findAll('div',{'class':re.compile('.*?a-box.*?')}, recursive=False)

                            shipments.pop(0)

                            for shipment in shipments:

                                shipdiv = shipment.find('div', {'class':'a-box-inner'})

                                sdivs = shipment.div.div.findAll('div', recursive=False)

                                orderStatus = sdivs[0].span.text.strip()

                                status = self._getStatusFromDetailedStatus(orderStatus)

                                if status == MStore.ORDER_DELIVERED:

                                    deliveredOn = sdivs[0].findAll('span')[-1].text.strip()

                                    try:

                                        deliveredOn = deliveredOn.split(":")[1].strip()

                                    except:

                                        deliveredOn = "" 

                                deliveryestimatespan = sdivs[0].find('span', {'class':'a-color-success'})

                                deliveryEstimate = None

                                if deliveryestimatespan is not None:

                                    deliveryEstimate = deliveryestimatespan.find('span', {'class':'a-text-bold'}).text.strip()

                                productDivs = shipdiv.find('div', {'class':re.compile('.*?a-spacing-top-medium.*?')}).find('div', {'class':'a-row'}).findAll('div', recursive=False)

                                trackingUrl = None

                                for buttonDiv in shipdiv.findAll('span', {'class':'a-button-inner'}):

                                    if buttonDiv.find('a').text.strip()=='Track package':

                                        trackingUrl = buttonDiv.find('a')['href'].strip()

                                        if not trackingUrl.startswith("http"):

                                            trackingUrl = "http://www.amazon.in" + trackingUrl

                                        break

                                for prodDiv in productDivs:

                                    prodDiv.find('div', {'class':'a-fixed-left-grid-inner'})

                                    productTitle = prodDiv.find('div', {'class':'a-fixed-left-grid-inner'}).find("div", {'class':'a-row'}).find('a').text.strip()

                                    imgUrl = prodDiv.find("img")["src"]

                                    for subOrder in activeOrder['subOrders']:

                                        if subOrder['productTitle'] == productTitle:

                                            findMap = {"orderId": activeOrder['orderId'], "subOrders.merchantSubOrderId": subOrder.get("merchantSubOrderId")}

                                            updateMap = {}

                                            closedStatus = False

                                            updateMap['subOrders.$.imgUrl'] = imgUrl

                                            updateMap['subOrders.$.lastTracked'] = timestamp

                                            updateMap['subOrders.$.detailedStatus'] = orderStatus

                                            cashbackStatus = subOrder.get("cashBackStatus")

                                            updateMap['subOrders.$.status'] = status 

                                            if status==MStore.ORDER_DELIVERED:                               

                                                updateMap['subOrders.$.deliveredOn'] = deliveredOn

                                                closedStatus = True

                                                updateMap['subOrders.$.closed'] = True

                                                if cashbackStatus == Store.CB_PENDING:

                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_APPROVED

                                            if status==MStore.ORDER_CANCELLED:     

                                                closedStatus = True

                                                updateMap['subOrders.$.closed'] = True

                                                if cashbackStatus == Store.CB_PENDING:

                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_CANCELLED

                                            if status==MStore.ORDER_SHIPPED:   

                                                updateMap['subOrders.$.estimatedDeliveryDate'] = deliveryEstimate

                                                if trackingUrl is not None:

                                                    updateMap['subOrders.$.trackingUrl'] = trackingUrl

                                            if not closedStatus:

                                                closed = False

                                            bulk.find(findMap).update({'$set' : updateMap})

                                            break

                            bulk.find({'orderId': activeOrder['orderId']}).update({'$set':{'closed': closed}})            

                bulk.execute()

                return 'PARSED_SUCCESS'

            else:

                merchantOrderId = re.findall(r'https://www.amazon.in/gp/css/summary/edit.html\?orderID=(.*)?', url, re.IGNORECASE)[0] 

                merchantOrder = self.db.merchantOrder.find_one({"merchantOrderId":merchantOrderId})

                filename = directory + "/" + merchantOrderId   

                f = open(filename,'w')

                f.write(rawHtml) # python will convert \n to os.linesep

                f.close() # you can omit in most cases as the destructor will call if

                result = self.parseOrderRawHtml(merchantOrder['orderId'], merchantOrder['subTagId'], merchantOrder['userId'], rawHtml, url, True)['result']

                print "result", result

                try:

                    order1 = session.query(OrdersRaw).filter_by(id=merchantOrder['orderId']).first()

                    order1.status = result

                    order1.rawhtml = rawHtml

                    session.commit()

                except:

                    traceback.print_exc()

                finally:

                    session.close()

                return 'PARSED_SUCCESS'

                pass

            return 'PARSED_SUCCESS_NO_ORDERS'

        except:

            traceback.print_exc()    

            return 'PARSED_FAILED'

    def _getStatusFromDetailedStatus(self, detailedStatus):

        if "ordered from" in detailedStatus.lower():

            return MStore.ORDER_PLACED 

        for key, value in self.orderStatusRegexMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", "Store:", self.store_id

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)

    def scrapeAffiliate(self, startDate=None, endDate=None):

        br = getBrowserObject()

        br.add_password('https://assoc-datafeeds-eu.amazon.com', 'Saholic', 'Fnubyvp')

        url = AMAZON_AFF_URL

        response = br.open(url)

        #get data for past 40 days and store it to mongo

        dt = datetime.now()

        dat = dt - timedelta(days=2)

        url = AMAZON_AFF_FILE_URL%(datetime.strftime(dat, "%Y%m%d"))

        response = br.open(url)

        page = gzip.GzipFile(fileobj=response, mode='rb').read()

        j=-1

        for row in page.split("\n"):

            j += 1

            if j== 0 or j==1:

                continue

            fields = row.split("\t")

            if len(fields)>1:

                print fields

                amazonAffiliate = AmazonAffiliateInfo(fields[0], fields[1], fields[2], fields[3], fields[4], fields[5], fields[6], fields[7], fields[8], fields[9])

                print amazonAffiliate

                self.db.amazonAffiliateInfo.insert(todict(amazonAffiliate))

            else:

                break

    def parseTrackingUrl(self, trackingUrl):

        subOrder = {}

        page = fetchResponseUsingProxy(trackingUrl)

        soup = BeautifulSoup(page)

        alertContainer = soup.find("div", {"class":"a-box-inner a-alert-container"})

        if alertContainer is not None:

            statusText = alertContainer.h4.span.text

            detailedStatus = statusText.split(":")[0].strip()

            subOrder['detailedStatus'] = detailedStatus

            if detailedStatus.lower() == "in transit":

                pass

            elif detailedStatus.lower() == "undeliverable":

                subOrder['status'] = MStore.ORDER_CANCELLED

                return subOrder

        else:

            summaryLeft = soup.find(id="summaryLeft")

            statusText = summaryLeft.h2.text

            detailedStatus = statusText.split(":")[0].strip()

            subOrder['detailedStatus'] = detailedStatus

            if detailedStatus.lower() == "delivered":

                subOrder['deliveredOn'] = summaryLeft.findAll("span")[-2].text.strip()

                subOrder['status'] = MStore.ORDER_DELIVERED

                return subOrder

            else:

                subOrder['expectedDelivery'] = summaryLeft.findAll("span")[-1].text.strip()

        return subOrder

def main():

    #str1 = readSSh("/AmazonTrack/User533/403-3748792-1516356")

    #readSSh("/tmp/User2/404-2225153-7073122")

    store = getStore(1)

    #store.scrapeStoreOrders()

#    br = mechanize.Browser()

#    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),

#                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),

#                     ('Accept-Encoding', 'gzip,deflate,sdch'),                  

#                     ('Accept-Language', 'en-US,en;q=0.8'),                     

#                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]

#    store.scrapeStoreOrders()

    #store.parseOrderRawHtml(3418, "13232", 14, readSSh("/home/amit/3418"), "https://www.amazon.in/gp/css/summary/edit.html?orderID=402-1853822-0778736")

    #store.trackOrdersForUser(4355, "https://www.amazon.in/gp/css/summary/edit.html?orderID=171-4824011-7090713", readSSh("~/4355"))

    #readSSh("/tmp/User211/2015-04-12 10:32:41.905765")

    #store.scrapeAffiliate()

    parseDetailNotCreated()

def parseDetailNotCreated():

    try:

        store=getStore(1)

        orders = session.query(OrdersRaw).filter_by(status='DETAIL_NOT_CREATED_UNKNOWN').all()

        session.close()

        for order in orders:

            store.trackOrdersForUser(order.id, order.order_url, order.rawhtml)

    finally:

        session.close()

def readSSh(fileName):

    try:

        str1 = open(fileName).read()

        return str1

    except:

        ssh_client = SSHClient()

        str1 = ""

        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        ssh_client.connect('dtr', 22, 'root', 'ecip$dtrMay2014')

        sftp_client = ssh_client.open_sftp()

        try:

            if not os.path.exists(os.path.dirname(fileName)):

                os.makedirs(os.path.dirname(fileName))

            sftp_client.get(fileName, fileName)

            try:

                str1 = open(fileName).read()

                return str1

            finally:

                pass

        except:

            "could not read"

        return str1

if __name__ == '__main__':

        main()