Subversion Repositories SmartDukaan

# coding=utf-8

'''

Created on Jan 15, 2015

@author: amit

'''

from base64 import encode

from bs4 import BeautifulSoup

from datetime import datetime, timedelta, date

from dtr.api.Order import process_rejects

from dtr.dao import Order, SubOrder, AmazonAffiliateInfo

from dtr.main import getStore, Store as MStore, ParseException, getBrowserObject, \

    ungzipResponse, tprint

from dtr.sources.flipkart import todict, AFF_REPORT_URL

from dtr.storage.DataService import OrdersRaw, Orders, Order_Parse_Info,\

    All_user_addresses

from dtr.storage.Mongo import getDealRank

from dtr.utils.utils import fetchResponseUsingProxy, readSSh

from elixir import *

import base64

import dtr

import gzip

import mechanize

import os.path

import re

import time

import traceback

import urllib2

ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'

ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'

THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'

AMAZON_AFF_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/listReports'

AMAZON_AFF_FILE_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/getReport?filename=saholic-21-orders-report-%s.tsv.gz'

class Store(MStore):

    orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],

                            MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],

                            MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],

                            MStore.ORDER_DELIVERED : ['delivered', 'your package was delivered', 'package was handed directly to customer']

                           }

    def __init__(self,store_id):

        super(Store, self).__init__(store_id)

    def getName(self):

        return "amazon"

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl, track=False):

        parseString = "Tracking" if track else "Transacted"

        print parseString, "Order Id to be parsed is :", orderId

        resp = {}

        resp['result'] = 'ORDER_NOT_CREATED'

        if ORDER_SUCCESS_URL in orderSuccessUrl or THANKYOU_URL in orderSuccessUrl:

            try:

                soup = BeautifulSoup(rawHtml)

                try:

                    orderUrl = soup.find('div', {"id":"thank-you-box-wrapper"}).div.findAll('div', recursive=False)[1].a['href']

                    merchantOrderId = re.findall(r'.*&oid=(.*)&?.*?', orderUrl)[0]

                except:

                    merchantOrderId = soup.find(id="orders-list").div.span.b.text

                order = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl, True)

                order.orderTrackingUrl = ORDER_REDIRECT_URL % (merchantOrderId)

                order.orderSuccessUrl = orderSuccessUrl

                order.merchantOrderId = merchantOrderId

                order.requireDetail = True

                order.status = 'html_required'

                order.closed = None

                if self._saveToOrder(todict(order)):

                    resp['result'] = 'ORDER_CREATED'

                    resp["url"] = ORDER_REDIRECT_URL % (merchantOrderId)

                    resp["htmlRequired"] = True

                    resp['orderId'] = orderId

                else:

                    resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            except:

                #Write all cases here for Order Not created Known

                try:

                    if not soup.body:

                        resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                    elif 'Securely redirecting you' in soup.h3.text.strip() or soup.h3.text.strip()=="Orders":

                        resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                    else:

                        raise

                except:

                    try:

                        if soup.h1.text.strip() in ['This is a duplicate order', 'There was a problem with your payment.', 'Your Orders', 'Your Shopping Cart is empty.', 'Select a payment method', 'Edit quantities'] or "Saved for later" in soup.h1.text.strip():

                            resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                        else:

                            raise

                    except:

                        try:

                            if soup.h2.text.strip() in ['Web page not available','Webpage not available']:

                                resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                            else:

                                raise

                        except:

                            try:

                                if soup.find(id="loading-spinner-img") is not None or soup.find(id="anonCarousel1") is not None or soup.find(id="ap_signin_pagelet_title") is not None or soup.find(id="nav-greeting-name") is not None:

                                    resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                                elif soup.find("b", {'class':'h1'}).text.strip().find("We're sorry") > -1:

                                    resp['result'] = 'ORDER_NOT_CREATED_KNOWN'

                                else:

                                    raise

                            except:

                                resp['result'] = 'ORDER_NOT_CREATED_UNKNOWN'

        else:

            try:

                mo = self.db.merchantOrder.find_one({"orderId":orderId})

                if mo is not None:

                    merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl, False)

                    merchantOrder.createdOn = mo.get("createdOn")

                    merchantOrder.createdOnInt =  mo.get("createdOnInt")

                else:

                    print "Could not find amazon order with order Id", orderId

                    merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

                soup = BeautifulSoup(rawHtml)

                try:

                    self.parseNewStlye(merchantOrder, soup)

                    resp['result'] = 'DETAIL_CREATED'

                except:

                    try:

                        traceback.print_exc()

                        self.parseOldStlye(merchantOrder, soup)

                        resp['result'] = 'DETAIL_CREATED'

                    except:

                        traceback.print_exc()

                        try:

                            self.parseCancelled(merchantOrder, soup)

                            resp['result'] = 'ORDER_CANCELLED'

                        except:

                            try:

                                if soup.h1.span.text=="Account" or soup.h1.text.strip() in ["Your Account"]:

                                    resp['result'] = 'DETAIL_NOT_CREATED_KNOWN'

                                else:

                                    raise

                            except:

                                if soup.find(id="ap_signin_pagelet_title").h1.text.strip()=="Sign In":

                                    resp['result'] = 'DETAIL_NOT_CREATED_KNOWN'

                                else:

                                    raise

            except:

                order = self.db.merchantOrder.find_one({"orderId":orderId})

                if order is not None:

                    self.db.merchantOrder.update({"orderId":orderId}, {"$set":{"status":"parse_failed"}})

                print "Error occurred"

                resp['result'] = 'DETAIL_NOT_CREATED_UNKNOWN'

                traceback.print_exc()

        return resp    

    #This should be exposed from api for specific sources

    def scrapeStoreOrders(self):

        orders = self.db.merchantOrder.find({"storeId":1, "closed":False, "subOrders.closed":False, "subOrders.trackingUrl":{"$exists":True}})

        for merchantOrder in orders:

            executeBulk = False

            try:

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                closed = True

                map1 = {}

                for subOrder in merchantOrder.get("subOrders"):

                    if subOrder.get("closed"):

                        continue

                    elif subOrder.get("trackingUrl") is None:

                        closed = False

                        continue

                    findMap = {"orderId":merchantOrder.get("orderId"), "subOrders.merchantSubOrderId":subOrder.get("merchantSubOrderId")} 

                    trackingUrl = subOrder.get("trackingUrl")

                    if not map1.has_key(trackingUrl):

                        map1[trackingUrl] = self.parseTrackingUrl(trackingUrl, merchantOrder.get("orderId"))

                    newOrder = map1.get(trackingUrl)

                    if newOrder:

                        executeBulk = True

                        updateMap = self.getUpdateMap(newOrder, subOrder.get('cashBackStatus'))

                        print findMap, "\n", updateMap

                        bulk.find(findMap).update({'$set' : updateMap})

                        closed = closed and newOrder['closed']

                if executeBulk:

                    bulk.find({"orderId":merchantOrder.get("orderId")}).update({"$set":{"closed":closed, "parseError":False}})

                    bulk.execute()

            except:

                tprint("Could not update " + str(merchantOrder['orderId']) + " For store " + self.getName())

                self.db.merchantOrder.update({"orderId":merchantOrder['orderId']}, {"$set":{"parseError":True}})

                traceback.print_exc()

    def parserest(self, soup):

        print "Hi"

        if soup.find('h1'):

            print "OK"

    def parseOldStlye(self, merchantOrder, soup):

        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl

        table = soup.body.findAll("table", recursive=False)[1]

        #print table

        tables = table.tr.td.findAll("table", recursive=False)

        for tr in tables[2].findAll("tr"):

            boldElement = tr.td.b

            if "Order Placed" in str(boldElement):

                merchantOrder.placedOn = boldElement.next_sibling.strip()

            if "order number" in str(boldElement):

                merchantOrder.merchantOrderId = boldElement.next_sibling.strip()

            if "Order Total" in str(boldElement):

                merchantOrder.paidAmount = int(float(boldElement.find('span').contents[-1].replace(',','')))

        anchors = table.tr.td.findAll("a", recursive=False)

        paymentAnchor = anchors.pop(-1)

        count = 0

        subOrders = []

        merchantOrder.subOrders = subOrders

        counter = 0 

        for anchor in anchors:

            count += 1

            tab = anchor.next_sibling

            status = MStore.ORDER_PLACED

            subStr = "Delivery #" + str(count) + ":"

            if subStr in  tab.find("b").text:

                detailedStatus = tab.find("b").text.replace(subStr, '').strip()

            tab = tab.next_sibling.next_sibling

            trs = tab.find("table").find('tbody').findAll("tr", recursive = False)

            estimatedDelivery = trs[0].td.find("b").next_sibling.strip()

            orderItemTrs = trs[1].findAll("td", recursive=False)[1].table.tbody.findAll("tr", recursive = False)

            i = -1

            for orderItemTr in orderItemTrs:

                i += 1

                if i%2 == 0:

                    continue

                counter += 1

                quantity =  int(re.findall(r'\d+', orderItemTr.td.contents[0])[0])

                productUrl = orderItemTr.td.contents[1].a["href"]

                productTitle = orderItemTr.td.contents[1].a.text

                unitPrice = int(float(orderItemTr.findAll('td')[1].span.text.replace('Rs. ','').replace(',','')))

                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, unitPrice*quantity, status, quantity)

                subOrder.merchantSubOrderId = str(counter) + " of " + merchantOrder.merchantOrderId

                subOrder.estimatedDeliveryDate = estimatedDelivery

                estDlvyTime = datetime.strptime(estimatedDelivery.split('-')[0].strip(), "%A %d %B %Y")

                createdOn = datetime.fromtimestamp(merchantOrder.createdOnInt)

                subOrder.trackAfter = int(time.mktime(max(estDlvyTime-timedelta(days=4),createdOn + timedelta(days=3)).timetuple()))

                subOrder.productCode = productUrl.split('/')[5]

                subOrder.detailedStatus = detailedStatus

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, unitPrice*quantity)

                dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

                subOrder.dealRank = dealRank.get('rank')

                subOrder.rankDesc = dealRank.get('description')

                subOrder.maxNlc = dealRank.get('maxNlc')

                subOrder.minNlc = dealRank.get('minNlc')

                subOrder.db = dealRank.get('dp')

                subOrder.itemStatus = dealRank.get('status')

                cashbackStatus = Store.CB_PENDING

                if cashbackAmount <= 0:

                    cashbackStatus = Store.CB_NA

                subOrder.cashBackStatus = cashbackStatus

                subOrder.cashBackAmount = cashbackAmount

                if percentage > 0:

                    subOrder.cashBackPercentage = percentage

                subOrders.append(subOrder)

        priceList = paymentAnchor.next_sibling.next_sibling.next_sibling.table.table.tbody.tbody.tbody.findAll('tr', recursive=False)

        totalAmount = 0

        grandAmount = 0

        for price in priceList:

            labelTd = price.td

            if 'Subtotal:' in labelTd.text:

                totalAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))

            elif 'Grand Total:' in labelTd.text:

                grandAmount += int(float(labelTd.next_sibling.next_sibling.find('span').contents[-1].replace(',','')))

        if grandAmount < totalAmount:

            diff = totalAmount - grandAmount

            for subOrder in merchantOrder.subOrders:

                subOrder.amountPaid -= int(diff*(1-subOrder.amountPaid/totalAmount))

        merchantOrder.status='success'

        self._updateToOrder(todict(merchantOrder))

    def parseNewStlye(self, merchantOrder, soup):

        merchantOrder.orderTrackingUrl = merchantOrder.orderSuccessUrl

        orderDetailsContainer = soup.body.find(id="orderDetails")

        divAfterH1 = orderDetailsContainer.h1.next_sibling.next_sibling

        orderLeftDiv = divAfterH1.div

        placedOnSpan = orderLeftDiv.find("span", {'class':'order-date-invoice-item'})

        merchantOrder.placedOn =placedOnSpan.text.split('Ordered on')[1].strip()

        merchantOrder.merchantOrderId = placedOnSpan.next_sibling.next_sibling.text.split('Order#')[1].strip()

        try:

            priceBox = divAfterH1.next_sibling.next_sibling.next_sibling.next_sibling.find("div", {"class":"a-box-inner"}).div.div.findAll('div', recursive=False)[-1]

        except:

            priceBox = divAfterH1.next_sibling.next_sibling.next_sibling.next_sibling.find("div", {"class":"a-box a-last"}).div.div.findAll('div', recursive=False)[-1]

        priceRows = priceBox.findAll('div', {'class':'a-row'})

        subTotal = 0

        shippingPrice = 0   

        promoApplied = 0

        for priceRow in priceRows:

            if "Item(s) Subtotal:" in str(priceRow):

                subTotal = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Shipping:" in str(priceRow):

                shippingPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Grand Total:" in str(priceRow):

                grandPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

                merchantOrder.paidAmount = grandPrice

            elif "Total:" in str(priceRow):

                totalPrice  = int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

            elif "Promotion Applied:" in str(priceRow):

                promoApplied  += int(float(priceRow.div.next_sibling.next_sibling.span.span.text.replace('Rs.','').replace(',', '')))

        totalPaid = subTotal        

        if promoApplied > 0:

            totalPaid -= promoApplied

            if shippingPrice <= promoApplied:

                totalPaid  += shippingPrice

        shipmentDivs = orderDetailsContainer.find('div', class_='shipment').findAll('div', recursive = False)

        subOrders = []

        merchantOrder.subOrders = subOrders

        i=1

        closedStatus = True

        for shipmentDiv in shipmentDivs:

            innerBoxes = shipmentDiv.findAll('div', recursive = False)

            statusDiv = innerBoxes[0]

            subOrderStatus = statusDiv.div.span.text.strip()

            deliverySpan = statusDiv.div.div.find_next_sibling('div').span

            productDivs = innerBoxes[-1].div.div.div.findAll('div', recursive=False)

            subOrders = []

            merchantOrder.subOrders = subOrders

            for i, productDiv in enumerate(productDivs):

                i +=1 

                imgDiv  = productDiv.div.div

                detailDiv = imgDiv.find_next_sibling('div')

                detailDivs = detailDiv.findAll('div', recursive=False)

                arr = detailDivs[0].a.text.strip().split(" of ", 1)

                (productTitle, quantity) = (arr[-1], (1 if len(arr)==1 else int(arr[0])) )

                try:

                    unitPrice = int(float(detailDivs[2].span.text.replace('Rs. ','').replace(',','')))

                except:

                    unitPrice = int(float(detailDivs[3].span.text.replace('Rs. ','').replace(',','')))

                amountPaid = int((unitPrice*quantity*totalPaid)/subTotal)

                productUrl = "http://www.amazon.in" + detailDivs[0].a.get('href')

                subOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid, MStore.ORDER_PLACED, quantity)

                subOrder.productCode = productUrl.split('/')[5]

                subOrder.unitPrice = unitPrice

                subOrder.merchantSubOrderId = str(i) + " of " + merchantOrder.merchantOrderId

                estDlvyTime = datetime.now()

                if deliverySpan is not None:

                    try:

                        subOrder.estimatedDeliveryDate = deliverySpan.span.text.strip()

                        estDate = subOrder.estimatedDeliveryDate.split("-")[0].strip()

                        subOrder.estimatedDeliveryInt = int(time.mktime((datetime.strptime(estDate, "%A %d %B %Y")).timetuple()))

                        estDlvyTime = datetime.strptime(estDate, "%A %d %B %Y")

                    except:

                        if "Delivered on" in deliverySpan.text:

                            subOrder.deliveredOn = deliverySpan.text.split(":")[1].strip() 

                        subOrder.estimatedDeliveryDate = "Not available"

                createdOn = datetime.fromtimestamp(merchantOrder.createdOnInt)

                subOrder.trackAfter = int(time.mktime(max(estDlvyTime-timedelta(days=4),createdOn + timedelta(days=3)).timetuple()))

                subOrder.detailedStatus = subOrderStatus

                subOrder.deliveryCharges = shippingPrice

                subOrder.imgUrl = imgDiv.img["src"]

                (cashbackAmount, percentage) = self.getCashbackAmount(subOrder.productCode, amountPaid)

                dealRank = getDealRank(subOrder.productCode, self.store_id, merchantOrder.userId)

                subOrder.dealRank = dealRank.get('rank')

                subOrder.rankDesc = dealRank.get('description')

                cashbackStatus = Store.CB_PENDING

                if cashbackAmount <= 0:

                    cashbackStatus = Store.CB_NA

                subOrder.cashBackStatus = cashbackStatus

                subOrder.cashBackAmount = cashbackAmount

                if percentage > 0:

                    subOrder.cashBackPercentage = percentage

                if hasattr(subOrder, 'deliveredOn'):                               

                    subOrder.status = Store.ORDER_DELIVERED

                    subOrder.closed = True

                    if subOrder.cashBackStatus == Store.CB_PENDING:

                        subOrder.cashBackStatus = Store.CB_APPROVED

                elif closedStatus:

                    closedStatus= False

                subOrders.append(subOrder)

        merchantOrder.status='success'

        merchantOrder.closed = closedStatus

        self._updateToOrder(todict(merchantOrder))

    def parseCancelled(self, merchantOrder,soup):

        try:

            fonts = soup.body.findAll("table", recursive=False)[1].findAll("font")

            if fonts[0].text == "Important Message":

                if fonts[1].text=="This order has been cancelled.":

                    merchantOrder.closed = True

                    merchantOrder.status = "cancelled"

                    merchantOrder.requireDetail = False

                    self._updateToOrder(todict(merchantOrder))

                    return

                else:

                    raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

            else:

                raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

        except:

            orderDetails = soup.body.find(id="orderDetails")

            if orderDetails is not None and orderDetails.h4.text == "This order has been cancelled.":

                merchantOrder.closed = True

                merchantOrder.status = "cancelled"

                merchantOrder.requireDetail = False

                self._updateToOrder(todict(merchantOrder))

            else:

                raise ParseException("parseCancelled", "Found detailed status" + fonts[1].text)

    def getTrackingUrls(self, userId):

        missingOrderUrls = []

        missingOrders = self._getMissingOrders({'userId':userId})

        for missingOrder in missingOrders:

            missingOrderUrls.append(ORDER_REDIRECT_URL%(missingOrder['merchantOrderId']))

        orders = self._getActiveOrders({'userId':userId, "subOrders.trackingUrl":{"$exists":False} })

        count = len(orders)

        print "count", count

        print "Missing Urls"

        print "*************"

        print missingOrderUrls

        if count > 0:

            return missingOrderUrls + ['https://www.amazon.in/gp/css/order-history', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled', 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled&startIndex=10']

        else: 

            return missingOrderUrls

    def trackOrdersForUser(self, userId, url, rawHtml):

        directory = "/AmazonTrack/User" + str(userId)

        if not os.path.exists(directory):

            os.makedirs(directory)

        try:

            searchMap = {'userId':userId}

            collectionMap = {'merchantOrderId':1}

            activeOrders = self._getActiveOrders(searchMap, collectionMap)

            datetimeNow = datetime.now()

            timestamp = int(time.mktime(datetimeNow.timetuple()))

            print "url----------------", url

            if url == 'https://www.amazon.in/gp/css/order-history' or 'https://www.amazon.in/gp/css/order-history/?orderFilter=cancelled' in url:

                if url == 'https://www.amazon.in/gp/css/order-history':

                    filename = directory + "/orderSummary" + datetime.strftime(datetime.now(), '%d-%m:%H:%M:%S')

                else:   

                    filename = directory + "/cancelledSummary" + datetime.strftime(datetime.now(), '%d-%m:%H:%M:%S')   

                f = open(filename,'w')

                f.write(rawHtml) # python will convert \n to os.linesep

                f.close() # you can omit in most cases as the destructor will call if

                soup = BeautifulSoup(rawHtml)

                allOrders = soup.find(id="ordersContainer").findAll('div', {'class':'a-box-group a-spacing-base order'})

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                for activeOrder in activeOrders:

                    for orderEle in allOrders:

                        deliveredOn = None

                        deliveryEstimate = None

                        shippingEstimate = None

                        orderdiv = orderEle.find('div', {'class':'a-box a-color-offset-background order-info'}).find('div', {'class':'a-fixed-right-grid-col actions a-col-right'})

                        merchantOrderId = orderdiv.find('span', {'class':'a-color-secondary value'}).text.strip()

                        if merchantOrderId==activeOrder['merchantOrderId']:

                            closed = True

                            shipments = orderEle.findAll('div',{'class':re.compile('.*?a-box.*?')}, recursive=False)

                            shipments.pop(0)

                            for shipment in shipments:

                                shipdiv = shipment.find('div', {'class':'a-box-inner'})

                                sdivs = shipment.div.div.findAll('div', recursive=False)

                                try:

                                    orderStatus = sdivs[0].findAll('div')[1].div.text.strip()

                                except:

                                    orderStatus = sdivs[0].findAll('div')[1].text.strip()

                                print orderStatus

                                try:

                                    if "Dispatch estimate" in orderStatus:

                                        status = MStore.ORDER_PLACED

                                        shippingEstimate = orderStatus.split("Dispatch estimate").split("-")[0].strip()

                                    elif "Delivery estimate" in orderStatus:

                                        status = MStore.ORDER_SHIPPED

                                        deliveryEstimate = orderStatus.split("Delivery estimate").split("-")[0].strip()

                                    else:

                                        self._getStatusFromDetailedStatus(orderStatus)

                                except:

                                    closed=False

                                    status = None

                                if status == MStore.ORDER_DELIVERED:

                                    dateString = sdivs[0].span.text.strip().split("Delivered ")[1].strip()

                                    deliveredOn = datetime.strftime(getDateStringDelivered(dateString), '%d-%b-%y')

                                if status == MStore.ORDER_SHIPPED:

                                    dateString = sdivs[0].span.text.strip().split("Arriving ")[1].strip()

                                    deliveryEstimate = datetime.strftime(getDateStringArriving(dateString), '%d-%b-%y')

                                productDivs = shipdiv.find('div', {'class':re.compile('.*?a-spacing-top-medium.*?')}).find('div', {'class':'a-row'}).findAll('div', recursive=False)

                                trackingUrl = None

                                for buttonDiv in shipdiv.findAll('span', {'class':'a-button-inner'}):

                                    if buttonDiv.find('a').text.strip()=='Track package':

                                        trackingUrl = buttonDiv.find('a')['href'].strip()

                                        if not trackingUrl.startswith("http"):

                                            trackingUrl = "http://www.amazon.in" + trackingUrl

                                        break

                                for prodDiv in productDivs:

                                    prodDiv.find('div', {'class':'a-fixed-left-grid-inner'})

                                    productTitle = prodDiv.find('div', {'class':'a-fixed-left-grid-inner'}).find("div", {'class':'a-row'}).find('a').text.strip()

                                    imgUrl = prodDiv.find("img")["src"]

                                    for subOrder in activeOrder['subOrders']:

                                        if subOrder['productTitle'] in productTitle:

                                            findMap = {"orderId": activeOrder['orderId'], "subOrders.merchantSubOrderId": subOrder.get("merchantSubOrderId")}

                                            updateMap = {}

                                            closedStatus = False

                                            updateMap['subOrders.$.imgUrl'] = imgUrl

                                            updateMap['subOrders.$.lastTracked'] = timestamp

                                            if status:

                                                updateMap['subOrders.$.detailedStatus'] = status

                                                updateMap['subOrders.$.status'] = status 

                                                cashbackStatus = subOrder.get("cashBackStatus")

                                            if status==MStore.ORDER_DELIVERED:

                                                if not deliveredOn:                               

                                                    updateMap['subOrders.$.deliveredOn'] = deliveredOn

                                                closedStatus = True

                                                updateMap['subOrders.$.closed'] = True

                                                if cashbackStatus == Store.CB_PENDING:

                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_APPROVED

                                            if status==MStore.ORDER_CANCELLED:     

                                                closedStatus = True

                                                updateMap['subOrders.$.closed'] = True

                                                if cashbackStatus == Store.CB_PENDING:

                                                    updateMap['subOrders.$.cashBackStatus'] = Store.CB_CANCELLED

                                            if status==MStore.ORDER_SHIPPED:

                                                if deliveryEstimate:   

                                                    updateMap['subOrders.$.estimatedDeliveryDate'] = deliveryEstimate

                                                if trackingUrl is not None:

                                                    updateMap['subOrders.$.trackingUrl'] = trackingUrl

                                            if shippingEstimate:   

                                                updateMap['subOrders.$.estimatedShippingDate'] = shippingEstimate

                                            if not closedStatus:

                                                closed = False

                                            bulk.find(findMap).update({'$set' : updateMap})

                                            break

                            bulk.find({'orderId': activeOrder['orderId']}).update({'$set':{'closed': closed}})            

                bulk.execute()

                return 'PARSED_SUCCESS'

            else:

                merchantOrderId = re.findall(r'https://www.amazon.in/gp/css/summary/edit.html\?orderID=(.*)?', url, re.IGNORECASE)[0]

                print  "merchantOrderId", merchantOrderId

                merchantOrder = self.db.merchantOrder.find_one({"merchantOrderId":merchantOrderId})

                filename = directory + "/" + merchantOrderId   

                f = open(filename,'w')

                f.write(rawHtml) # python will convert \n to os.linesep

                f.close() # you can omit in most cases as the destructor will call if

                result = self.parseOrderRawHtml(merchantOrder['orderId'], merchantOrder['subTagId'], merchantOrder['userId'], rawHtml, url, True)['result']

                print "result", result

                try:

                    order1 = session.query(OrdersRaw).filter_by(id=merchantOrder['orderId']).first()

                    order1.status = result

                    order1.rawhtml = rawHtml

                    session.commit()

                except:

                    traceback.print_exc()

                finally:

                    session.close()

                return 'PARSED_SUCCESS'

                pass

            return 'PARSED_SUCCESS_NO_ORDERS'

        except:

            traceback.print_exc()    

            return 'PARSED_FAILED'

    def _getStatusFromDetailedStatus(self, detailedStatus):

        if "ordered from" in detailedStatus.lower():

            return MStore.ORDER_PLACED 

        for key, value in self.orderStatusRegexMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", "Store:", self.store_id, detailedStatus

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)

    def scrapeAffiliate(self, startDate=None, endDate=None):

        br = getBrowserObject()

        br.add_password('https://assoc-datafeeds-eu.amazon.com', 'Saholic', 'Fnubyvp')

        url = AMAZON_AFF_URL

        response = br.open(url)

        #get data for past 40 days and store it to mongo

        dt = datetime.now()

        dat = dt - timedelta(days=2)

        url = AMAZON_AFF_FILE_URL%(datetime.strftime(dat, "%Y%m%d"))

        response = br.open(url)

        page = gzip.GzipFile(fileobj=response, mode='rb').read()

        j=-1

        for row in page.split("\n"):

            j += 1

            if j== 0 or j==1:

                continue

            fields = row.split("\t")

            if len(fields)>1:

                print fields

                amazonAffiliate = AmazonAffiliateInfo(fields[0], fields[1], fields[2], fields[3], fields[4], fields[5], fields[6], fields[7], fields[8], fields[9])

                print amazonAffiliate

                self.db.amazonAffiliateInfo.insert(todict(amazonAffiliate))

            else:

                break

    def parseTrackingUrl(self, trackingUrl, orderId):

        print trackingUrl

        subOrder = {}

        page = fetchResponseUsingProxy(trackingUrl)

        status = MStore.ORDER_SHIPPED

        #print page

        soup = BeautifulSoup(page)

        header1 = soup.find("h1") 

        if header1:

            if header1.text=="Sign In":

                print "Login page is displayed for order id", orderId

                return subOrder

        try:

            print "Tracking page is displayed for order id", orderId

            detailedStatus = soup.find("div", {"class":"top"}).span.text.strip()

            try:

                displayStatus = soup.find("div",{"class":"a-column a-span12 shipment-status-content"}).span.text.strip()

            except:

                displayStatus = detailedStatus

            print displayStatus

            if detailedStatus.lower().find("delivered")>=0:

                print detailedStatus

                displayStatus = "Delivered"

                status = "Delivered"

                try:

                    subOrder["deliveredOn"] = detailedStatus.split("on")[1].strip()

                except:

                    pass

            elif detailedStatus.lower() == 'returned':

                status = 'Cancelled'

            subOrder['status'] = status

            subOrder['detailedStatus'] = displayStatus 

        except:

            print "failed to parse", orderId

            traceback.print_exc()

        return subOrder

    def parseInfo(self,):

        from pyquery import PyQuery as pq

        orders = list(session.query(Orders).filter_by(store_id=self.store_id).filter_by(status='DETAIL_CREATED').group_by(Orders.user_id).all())

        try:

            for order in orders:

                try:

                    doc = pq(order.rawhtml)

                    #a1= " ".join(["" if not div.text else div.text.replace("\t","").replace("\n","").replace(" ", "") for div in pq(doc('article')[-1])('div')])

                    lists = doc('ul.displayAddressUL li')

                    orderInfo = All_user_addresses()

                    orderInfo.address = lists[-3].text

                    orderInfo.user_id = order.user_id

                    orderInfo.source = 'order' 

                    #orderInfo.order_id = order.id

                    #orderInfo.email = None

                    #orderInfo.name = lists[0].text 

                    #orderInfo.mobile = None

                    adSplit = lists[-2].text.split(",")

                    match = re.match(r"([a-z ]+)([0-9]+)", adSplit[1], re.I)

                    if match:

                        items = match.groups()

                    orderInfo.city = adSplit[0].strip()

                    orderInfo.pincode = items[1].strip()

                    orderInfo.state = items[0].strip().title()

                    session.commit()

                except:

                    session.rollback()

                    continue

        finally:

            session.close()

def main():

    #getSummaryFile("/AmazonTrack/User8703/")

    #process_rejects("/home/amit/Downloads/RejectCashbackTemplate.xlsx")

    #str1 = readSSh("/AmazonTrack/User2466/orderSummary28-06:13:25:08")

    #str1 = readSSh("/AmazonTrack/User2466/orderSummary24-06:17:38:04")

    #str1 = readSSh("/AmazonTrack/User2466/orderSummary24-06:17:38:01")

    #str1 = readSSh("/AmazonTrack/User2466/orderSummary24-06:17:31:53")

    #str1 = readSSh("/AmazonTrack/User2466/orderSummary28-06:13:25:08")

    #str1 = readSSh("/AmazonTrack/User2466/orderSummary28-06:13:25:08")

    #readSSh("/tmp/User2/404-2225153-7073122")

   # all={2769:'http://www.amazon.in/gp/css/shiptrack/view.html/ref=oh_aui_direct_track_pkg_o00_s00?ie=UTF8&marketplaceOfOrigin=&orderID=402-3378626-7402755&orderingShipmentId=759313961106&packageId=1&ref=',2924:'http://www.amazon.in/gp/css/shiptrack/view.html/ref=oh_aui_direct_track_pkg_o04_s00?ie=UTF8&marketplaceOfOrigin=&orderID=404-1934145-4940333&orderingShipmentId=761351446106&packageId=1&ref=',3013:'http://www.amazon.in/gp/css/shiptrack/view.html/ref=oh_aui_direct_track_pkg_o03_s00?ie=UTF8&marketplaceOfOrigin=&orderID=404-2323646-9315538&orderingShipmentId=1157078607092&packageId=1&ref=',4804:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=ljokmpnqlonpq&orderId=171-6936083-5808328&packageIndex=0&shipmentId=Dc7LxLj8R',6447:'http://www.amazon.in/gp/css/shiptrack/view.html/ref=oh_aui_direct_track_pkg_o00_s00?ie=UTF8&marketplaceOfOrigin=&orderID=402-5267336-7658751&orderingShipmentId=796566362106&packageId=1&ref=',6764:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=ljnplqntnqmnq&orderId=403-1762134-8866741&packageIndex=0&shipmentId=D1Q77kmKR',6807:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkqjsvipqoptq&orderId=403-2423963-4963501&packageIndex=0&shipmentId=DVLdJkmgR',6823:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llljooomtkqvq&orderId=171-4857073-8237901&packageIndex=0&shipmentId=DVCh1YmQR',6887:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkqkkuprtlqvq&orderId=171-2588190-5695535&packageIndex=0&shipmentId=DVmbM1mJR',7054:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=ljrgnvllmrptq&orderId=171-1005907-4197952&packageIndex=0&shipmentId=DVG4nnmjR',7237:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llnoptjttjsnq&orderId=404-9151579-1801132&packageIndex=0&shipmentId=Dn8Ht6mFR',7457:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=ljnllolormovq&orderId=404-7441140-2089951&packageIndex=0&shipmentId=DRTdg5xBN',7740:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lmjkmopqkmsvq&orderId=403-3268269-3497105&packageIndex=0&shipmentId=DXMRKmmrR',8237:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lmjkrqnosrmvq&orderId=402-9929953-1629148&packageIndex=0&shipmentId=D09sZsxsN',8238:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llllooltlktpq&orderId=402-5577202-4121143&packageIndex=0&shipmentId=DgcwgQmrR',8595:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkkmrvhqqkqvq&orderId=403-6686659-8341932&packageIndex=0&shipmentId=DVxlhWmSR',8948:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=ljrnmwpplknpq&orderId=404-4274603-2058711&packageIndex=0&shipmentId=DN0JbVkpR',8979:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lknktuorqoqrq&orderId=403-7785786-1441940&packageIndex=0&shipmentId=DgprHkkwR',9184:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkjkswoprrlpq&orderId=404-1980439-1674729&packageIndex=0&shipmentId=DwpFBGkzR',9207:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkrnrwqnmllpq&orderId=402-2502378-3282757&packageIndex=0&shipmentId=D6GzwWkdR',9285:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=ljrnqwnqtmsrq&orderId=402-6126669-4675512&packageIndex=0&shipmentId=D12qBGk4R',9317:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkllqwqumnonq&orderId=404-6368449-3713921&packageIndex=0&shipmentId=DnQQynkYR',9324:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkknpsmntintq&orderId=171-1046230-1213920&packageIndex=0&shipmentId=DVQ7ymkXR',9431:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkknrpmtplkvq&orderId=404-2574261-0426710&packageIndex=0&shipmentId=DwXFY7kdR',9478:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkrolsqqnrmnq&orderId=403-4677805-1690750&packageIndex=0&shipmentId=DXdmD1kbR',9638:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkjlrtklqrsvq&orderId=171-1377835-5105921&packageIndex=0&shipmentId=DGcWzZkkR',9682:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llqmnxhsrmtpq&orderId=171-8612308-2068349&packageIndex=0&shipmentId=D68pzLkQR',9687:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkmjruqnpisvq&orderId=171-7128133-7135565&packageIndex=0&shipmentId=DVFsz7ktR',10997:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lllopvkrspsrq&orderId=404-2248175-7727520&packageIndex=0&shipmentId=DCScLlkVR',11163:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkmkrtklkklpq&orderId=404-5372353-5006730&packageIndex=0&shipmentId=DJg71dkyR',11186:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llqnntlsrokvq&orderId=403-6324939-6097130&packageIndex=0&shipmentId=DbvXwRJnN',11204:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=ljsnnroqposrq&orderId=404-3954569-0847544&packageIndex=0&shipmentId=Dc2PMzkgR',11238:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkngkqknsnovq&orderId=404-1477032-5612314&packageIndex=0&shipmentId=D61zJTklR',11343:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lknmlqmtljovq&orderId=403-5524974-9827564&packageIndex=0&shipmentId=DJkXxdk0R',11453:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkkoovkmrqkvq&orderId=403-5642828-3766719&packageIndex=0&shipmentId=DCntPFkjR',11591:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkolmumlpqqrq&orderId=171-3814201-4799568&packageIndex=0&shipmentId=DCG2tpkpR',11635:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lknmqoosoittq&orderId=404-9470010-7402734&packageIndex=0&shipmentId=DlQ3DtJPN',11640:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llqooropkjlpq&orderId=404-3232667-5995557&packageIndex=0&shipmentId=DPVnD3JDN',11744:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llmglsmrtpttq&orderId=402-4019238-8089904&packageIndex=0&shipmentId=DJNjd7k0R',11860:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkkotvntmomnq&orderId=402-7210209-6944332&packageIndex=0&shipmentId=DnJ0SlkpR',11868:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkklrsolmmrpq&orderId=402-2531669-1663506&packageIndex=0&shipmentId=DVzlS6kRR',12007:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkmlouqltkkvq&orderId=404-4771533-2973951&packageIndex=0&shipmentId=DntClfklR',12252:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkmhlqmtmoorq&orderId=404-9456048-1325927&packageIndex=0&shipmentId=DwxCpVkLR',12287:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lmjprxoqlmknq&orderId=402-8827174-6026750&packageIndex=0&shipmentId=D1RtCxk2R',12537:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkmhnulqprovq&orderId=404-4769504-7022766&packageIndex=0&shipmentId=DNv7XsktR',12585:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkkpsqqpojmnq&orderId=403-7086035-8821110&packageIndex=0&shipmentId=Dnxk8lknR',12633:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lmnosrklnqmnq&orderId=403-1160367-3817142&packageIndex=0&shipmentId=DgtZ89kZR',12636:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkompsjlrnonq&orderId=404-3266783-3721114&packageIndex=0&shipmentId=Dw1P8VkgR',12695:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llrjlqjonrkvq&orderId=171-7770062-0732332&packageIndex=0&shipmentId=DNfzgMkYR',12698:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llmplskprqtpq&orderId=403-9002502-2997950&packageIndex=0&shipmentId=Dncfg3kZR',12707:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkojoppqrrnpq&orderId=403-4036602-9350752&packageIndex=0&shipmentId=D19yglk0R',12709:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkplmujmslmrq&orderId=403-0058301-2031560&packageIndex=0&shipmentId=DnWPg7kHR',12757:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkplmwnnoisrq&orderId=404-1058705-4321945&packageIndex=0&shipmentId=DnZWqWkdR',12845:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llmpmvjnmrovq&orderId=171-6343010-3411517&packageIndex=0&shipmentId=DGT3qYk8R',12881:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkqppxoumkrpq&orderId=171-9174388-6523515&packageIndex=0&shipmentId=DwdvZTkZR',12892:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llrjmxmqtrqvq&orderId=404-4120934-8469951&packageIndex=0&shipmentId=D6lpZqkTR',12906:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkjpovnlmpqnq&orderId=171-4125889-1889940&packageIndex=0&shipmentId=DXw8ZJk7R',12915:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llqprqiupptpq&orderId=403-3197160-3069940&packageIndex=0&shipmentId=DJzhZYk3R',12916:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkmmmxpummqrq&orderId=403-6420484-5933945&packageIndex=0&shipmentId=DCzsZ6kCR',12917:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkqpqojtnpmnq&orderId=402-4411340-4496326&packageIndex=0&shipmentId=D6kcZPkZR',12922:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llmpnoqlnprpq&orderId=403-1132143-3111517&packageIndex=0&shipmentId=DJ6bZzksR',12925:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llqprqkmlikrq&orderId=403-8668894-6525135&packageIndex=0&shipmentId=DVQlZJkRR',12972:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lklgkvkusmkvq&orderId=404-0137476-1498700&packageIndex=0&shipmentId=DJltjQk9R',12980:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkojqonsplmvq&orderId=404-3833239-9846705&packageIndex=0&shipmentId=D6lgj0k9R',12983:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkomruounrqrq&orderId=404-3320980-8441119&packageIndex=0&shipmentId=DMH6v4JYN',13007:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkplouinnpovq&orderId=402-0535763-2745962&packageIndex=0&shipmentId=DGBWHV3sR',13021:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llqpruhlrjkvq&orderId=402-8874256-6725968&packageIndex=0&shipmentId=D6ZljckdR',13026:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkplouknqlmnq&orderId=403-9754892-7644334&packageIndex=0&shipmentId=DG9CjqkTR',13033:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkomrwoomlrtq&orderId=402-1262452-1058755&packageIndex=0&shipmentId=D1bdjfkLR',13071:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkmhqvhopnqvq&orderId=403-7740399-1427531&packageIndex=0&shipmentId=Dcn8fWkXR',13104:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkplprkrsnltq&orderId=403-5474992-4257913&packageIndex=0&shipmentId=DC6tfVk1R',13121:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llqpsslmqnntq&orderId=402-9481155-4197915&packageIndex=0&shipmentId=Dng9fXkbR',13122:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkmhrphpkjorq&orderId=402-7287272-2137155&packageIndex=0&shipmentId=DNgpHj3VR',13131:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkshrxntppnpq&orderId=402-3547176-4158757&packageIndex=0&shipmentId=D31Rh7k8R',13161:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llrjornsnmtpq&orderId=404-0451908-9529119&packageIndex=0&shipmentId=DP7WvnJ8N',13223:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkqpruhpqiqrq&orderId=403-8450087-2700305&packageIndex=0&shipmentId=DNxth6knR',13224:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkonlxjrtmonq&orderId=403-4869051-2325127&packageIndex=0&shipmentId=D1xhhXkSR',13250:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkomtslttpsrq&orderId=404-2537009-6415557&packageIndex=0&shipmentId=D31WhrkjR',13380:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llqptqmtmpkrq&orderId=404-3722195-9238730&packageIndex=0&shipmentId=DNksRV33R',13470:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llrgkplmkqnpq&orderId=171-0897884-0532300&packageIndex=0&shipmentId=D1wxRC32R',13575:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lknprrinmkltq&orderId=402-9225456-2338705&packageIndex=0&shipmentId=DS6FcbJdN',13667:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkonormlrkmvq&orderId=403-4129374-6176330&packageIndex=0&shipmentId=DV7zFl35R',13795:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lknnppqmomkvq&orderId=404-0117880-4748356&packageIndex=0&shipmentId=DJypk73wR',13914:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llrjtxhttilpq&orderId=171-0960593-7418702&packageIndex=0&shipmentId=DChSHc39R',13921:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llngkpktsmptq&orderId=403-2864741-1405962&packageIndex=0&shipmentId=DcbvHv3lR',13938:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lknnrqkoolntq&orderId=402-7351523-5146761&packageIndex=0&shipmentId=DGkVHD38R',14142:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llrgqumlpqsrq&orderId=402-5647686-2086731&packageIndex=0&shipmentId=DXqLBW3DR',14162:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llrknpmlqjrpq&orderId=404-8441935-9383513&packageIndex=0&shipmentId=DVMpBg3ZR',14315:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=llrknxoosnonq&orderId=403-0899217-3921942&packageIndex=0&shipmentId=D16qwd3FR',14500:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lmkolvosslkrq&orderId=403-4437734-8835562&packageIndex=0&shipmentId=D6vqYD32R',14513:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkpmsthnkjnpq&orderId=402-0499221-8384341&packageIndex=0&shipmentId=DPbFLJJ3N',14599:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkngrqnmmmovq&orderId=404-8192799-8830754&packageIndex=0&shipmentId=DnxvF63HR',14999:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkqmkqmmrqqvq&orderId=403-4817762-5489935&packageIndex=0&shipmentId=DGZ5VT3gR'}

   # all ={11642:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=lkoimxjprionq&orderId=404-8168821-8300356&packageIndex=0&shipmentId=D0tJDZJhN', 7794:'http://www.amazon.in/gp/your-account/ship-track/ref=oh_aui_st_v2_btn?ie=UTF8&itemId=ljqmlqkmroonq&orderId=402-4679832-9733904&packageIndex=0&shipmentId=DyxpqQxBN'}

    #for orderId, trackingUrl in all.iteritems():

    store = getStore(1)

     #   print store.parseTrackingUrl(trackingUrl, orderId)

    #store.scrapeStoreOrders()

#    br = mechanize.Browser()

#    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),

#                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),

#                     ('Accept-Encoding', 'gzip,deflate,sdch'),                  

#                     ('Accept-Language', 'en-US,en;q=0.8'),                     

#                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]

#    store.scrapeStoreOrders()

    #store.parseInfo()

    #store.parseOrderRawHtml(9274, "13232", 14, readSSh("/home/amit/Downloads/9274.html"), "https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html/ref=ox_spc_place_order?ie=UTF8&fromAnywhere=1&hasWorkingJav")

    #store.trackOrdersForUser(4355, "https://www.amazon.in/gp/css/summary/edit.html?orderID=171-4824011-7090713", readSSh("~/4355"))

    #readSSh("/tmp/User211/2015-04-12 10:32:41.905765")

    #store.scrapeAffiliate()

    #parseDetailNotCreated()

    #parseOrderNotCreated()

    for userId in store.db.merchantOrder.find({"storeId":1, "subOrders.closed":False}).distinct("userId"):

        userId = int(userId)

        directory = "/AmazonTrack/User" + str(userId) + "/"

        summaryFile=getSummaryFile(directory)

        if summaryFile is None:

            print "No summary for", userId

        else:

            print userId, directory+summaryFile

            store.trackOrdersForUser(userId, 'https://www.amazon.in/gp/css/order-history', readSSh(directory + summaryFile))

    #store.trackOrdersForUser(8703, 'https://www.amazon.in/gp/css/order-history', readSSh('/home/amit/Downloads/orderSummary06-10_12_15_54'))

def getSummaryFile(directory):

    date1 = datetime(2015,1,1)

    finalFile = None

    try:

        for file in os.listdir(directory):

            if file.startswith("orderSummary"):

                date2 = datetime.strptime("2015-" + file.split("orderSummary")[1].split(":")[0], "%Y-%d-%m")

                if date2 > date1:

                    date1 = date2

                    finalFile=file

    except:

        print "Missing directory"

    return finalFile 

def parseDetailNotCreated():

    try:

        store=getStore(1)

        orders = session.query(OrdersRaw).filter_by(status='DETAIL_NOT_CREATED_UNKNOWN').all()

        session.close()

        for order in orders:

            store.trackOrdersForUser(order.id, order.order_url, order.rawhtml)

    finally:

        session.close()

def getDateStringDelivered(dateString='Monday'):

    print dateString

    if dateString.lower()=='today':

        return date.today()

    if dateString.lower()=='yesterday':

        return date.today() - timedelta(days=1)

    try:

        return datetime.strptime(dateString, '%d-%b-%y')

    except:

        try:

            #get Closest Date from today

            curDate = date.today()

            curTime = datetime(curDate.year, curDate.month, curDate.day)

            curYear = curDate.year

            prevYear = curYear - 1

            dateMax = datetime.strptime(dateString + " " + str(curYear), "%A, %d %b %Y")

            dateMin = datetime.strptime(dateString + " " + str(prevYear), "%A, %d %b %Y")

            if dateMax <= curTime:

                return dateMax

            else:

                return dateMin 

        except:

            try:

                days_of_week = ['sunday','monday','tuesday','wednesday',

                            'thursday','friday','saturday']

                deltaDays = curDate.isoweekday() - days_of_week.index(dateString.lower())

                if deltaDays <= 0:

                    deltaDays= deltaDays + 7

                curDate = curDate - timedelta(days=deltaDays)

                print datetime.strftime(curDate, '%d-%b-%y')

                return curDate

            except:

                print "could not parse"

                return None

def getDateStringArriving(dateString='Thursday'):

    print dateString

    if dateString.lower()=='today':

        return date.today()

    if dateString.lower()=='tomorrow':

        return date.today() + timedelta(days=1)

    try:

        return datetime.strptime(dateString, '%d-%b-%y')

    except:

        try:

            #get Closest Date from today

            curDate = date.today()

            curTime = datetime(curDate.year, curDate.month, curDate.day)

            curYear = curDate.year

            nextYear = curYear + 1

            dateMin = datetime.strptime(dateString + " " + str(curYear), "%A, %d %b %Y")

            dateMax = datetime.strptime(dateString + " " + str(nextYear), "%A, %d %b %Y")

            if dateMin >= curTime:

                return dateMin

            else:

                return dateMax 

        except:

            try:

                days_of_week = ['sunday','monday','tuesday','wednesday',

                            'thursday','friday','saturday']

                deltaDays = days_of_week.index(dateString.lower()) - curDate.isoweekday()

                if deltaDays < 0:

                    deltaDays= deltaDays + 7

                curDate = curDate + timedelta(days=deltaDays)

                return curDate

            except:

                print "Could not parse"

                return None 

def parseOrderNotCreated():

    try:

        store=getStore(1)

        orders = session.query(OrdersRaw).filter_by(status='ORDER_NOT_CREATED_UNKNOWN').all()

        session.close()

        for order in orders:

            result = store.parseOrderRawHtml(order.id, order.sub_tag, order.user_id, order.rawhtml, order.order_url)['result']

            order1 = session.query(OrdersRaw).filter_by(id=order.id).first()

            order1.status = result

            session.commit()

    finally:

        session.close()

if __name__ == '__main__':

        readSSh("/AmazonTrack/User7091/orderSummary14-10:13:29:40")

        #main()

        #getDateStringArriving()