Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo

from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \

    Store as MStore, sourceMap, tprint

from dtr.storage.DataService import Clicks, Users, FlipkartOrders

from dtr.utils import utils

from dtr.utils.utils import fetchResponseUsingProxy, todict, getSkuData,\

    ORDER_PLACED

from elixir import *

from pprint import pprint

from pymongo.mongo_client import MongoClient

import StringIO

import csv

import hashlib

import importlib

import json

import mechanize

import pymongo

import re

import traceback

import urllib

import urllib2

from urlparse import urlparse

import zipfile

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

ORDER_TRACK_URL='https://m.flipkart.com/order_details?src=or&pr=1&type=physical&'

AFFILIATE_URL='https://affiliate.flipkart.com/'

AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'

AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"

AFF_STATUS_CANCELLED='cancelled'

AFF_STATUS_APPROVED='approved'

AFF_STATUS_DISAPPROVED='disapproved'

AFF_STATUS_PENDING='pending'

statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]

categoryMap = {3:"Mobiles", 5:"Tablets"}

class Store(MStore):

    OrderStatusMap = {

                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping'],

                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],

                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered'],

                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.']

                      }

    def __init__(self,store_id):

        client = MongoClient('mongodb://localhost:27017/')

        self.db = client.dtr

        super(Store, self).__init__(store_id)

    def getName(self):

        return "flipkart"

    def requestDownload(self):

    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"

        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

        br = getBrowserObject()

        br.set_debug_responses(True)

        br.open(AFFILIATE_URL)

        response = br.response()  # copy

        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]

        data = {

                    'j_username':'saholic1@gmail.com',

                    'j_password':'spice@2020'

            }

        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))

        till = datetime.strftime(date.today(),"%Y-%m-%d")

        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")

        #target = open("dowloadreportids", 'w')

        #target.truncate()

        for status in statuses:

            #try:

                data = {"till":till, "from":start, "filter":status}

                print json.dumps(data)

                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))

                print request

                response = utils.ungzipResponse(br.open(request))

                response = json.loads(response)

                print response

                if(response['status']=="OK"):

                    self.db.flipkartdownloadids.save(response)

                else:

                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")

                    #break 

            #except:

                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")

                #break 

    def scrapeStoreOrders(self,):

        orders = self._getActiveOrders()

        for order in orders:

            print "Order", self.store_name, order['orderId']

            try:

                closed = True

                url = ORDER_TRACK_URL + urlparse(order['orderSuccessUrl']).query

                page = fetchResponseUsingProxy(url)

                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

                sections = soup.findAll("div", {"class":"ui-app-card-body"})

                sections.pop(1)

                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

                fkSubOrders = mainOrder.findAll("li")

                #remove unwanted list

                fkSubOrders.pop(-1)

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                #fetching suborders details

                for subOrder in fkSubOrders:

                    updateMap = {}

                    y = subOrder.find("header").findAll("span")

                    for suborderId in y:

                        if "value emp" in str(suborderId):

                            merchantSubOrderId = suborderId.text

                            break

                    ul = subOrder.find("ul")

                    if ul is None:

                        ul = subOrder.findAll("div", recursive=False)[0].div.div

                    orderItems = ul.findAll("div", recursive=False)

                    i=0

                    for orderItem in orderItems:

                        closedStatus = False

                        merchantSubOrderId = merchantSubOrderId + str(i)

                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

                        if subOrder is None:

                            break

                        elif subOrder['closed']:

                            break

                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                        divs = orderItem.findAll('div', recursive=False)

                        orderTracking = divs[2]

                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})

                        orderTrackingDet = orderTrackingDetDiv.find('div',{'class':re.compile('tracking-remark')}).text

                        updateMap["subOrders.$.detailedStatus"] = orderTrackingDet

                        status = self._getStatusFromDetailedStatus(orderTrackingDet)

                        tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                        cashbackStatus = subOrder.get("cashBackStatus") 

                        if "approveDetails-complete" in str(tr):

                            if "processingDetails-complete" in str(tr):

                                if "shippingDetails-complete" in str(tr):

                                    if "delivery-complete" in str(tr):

                                        status = MStore.ORDER_DELIVERED

                                        if cashbackStatus == Store.CB_PENDING:

                                            cashbackStatus = Store.CB_APPROVED

                                        closedStatus = True

                                    else:     

                                        status = MStore.ORDER_SHIPPED

                                        courierTrackAnchor = orderTrackingDetDiv.find('a')

                                        trackingUrl = courierTrackAnchor("href")

                                        trackingText = courierTrackAnchor.text

                                        courierArr = trackingText.split(' : ')

                                        updateMap["subOrders.$.trackingUrl"] = trackingUrl

                                        updateMap["subOrders.$.trackingNumber"] = courierArr[1]

                                        updateMap["subOrders.$.courierName"] = courierArr[0]

                                else:

                                    status = MStore.ORDER_CANCELLED

                                    closedStatus = True

                                    if cashbackStatus == Store.CB_PENDING:

                                        cashbackStatus = Store.CB_CANCELLED

                            else: 

                                status = MStore.ORDER_PLACED

                        elif str(tr) in ["approveDetails-ongoing"]:

                            status=MStore.ORDER_PLACED             

                        if "dead" in str(tr) or status==MStore.ORDER_CANCELLED:

                            status = MStore.ORDER_CANCELLED

                            closedStatus = True

                            if cashbackStatus == Store.CB_PENDING:

                                cashbackStatus = Store.CB_CANCELLED

                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus

                        updateMap["subOrders.$.status"] = status

                        updateMap["subOrders.$.closed"] = closedStatus

                        if closed:

                            closed = closedStatus

                        bulk.find(findMap).update({'$set' : updateMap})

                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})

                bulk.execute()            

            except:

                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})

                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())

                traceback.print_exc()

    def scrapeAffiliate(self, deltaDays=0):

        if deltaDays is None:

            deltaDays=0

        endDate=date.today() - timedelta(days=1)

        startDate = endDate - timedelta(days=deltaDays)

        endDate = datetime.strftime(endDate, "%Y-%m-%d")

        startDate = datetime.strftime(startDate, "%Y-%m-%d")

        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=0"

        for status in statuses:

            nextUrl = url%(startDate, endDate, status)

            while nextUrl:

                req = urllib2.Request(nextUrl)

                nextUrl=''

                req.add_header('Fk-Affiliate-Id', 'saholic1g')

                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')

                resp = urllib2.urlopen(req)

                resString = json.loads(resp.read())

                orderList = resString["orderList"]

                if orderList:

                    for order in orderList:

                        order['sales'] = int(order['sales']['amount'])

                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])

                        subTagId = order.get("affExtParam1")

                        userId = None

                        email = None

                        if subTagId:

                            click = session.query(Clicks).filter_by(tag = subTagId).first()

                            if click is not None:

                                userId= click.user_id 

                                user = session.query(Users.email).filter_by(id = userId).first()

                                if user is not None:

                                    email = user.email

                        flipkartOrder = FlipkartOrders()

                        flipkartOrder.subtagId = subTagId

                        flipkartOrder.user_id = userId

                        flipkartOrder.identifier = order.get("identifier")

                        flipkartOrder.email = email

                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%d-%m-%Y %H:%M:%S")

                        flipkartOrder.status = order.get("status")

                        flipkartOrder.title = order.get("title")

                        flipkartOrder.price = order.get("price")

                        flipkartOrder.quantity = order.get("quantity")

                        flipkartOrder.productCode = order.get("productId")

                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")

                        flipkartOrder.payOut = order['tentativeCommission']

                        flipkartOrder.payOutPercentage = order['commissionRate']

                        skuData = getSkuData(2, order.get("productId"))

                        if skuData is not None:

                            flipkartOrder.catalogId = skuData.get("skuBundleId")

                            flipkartOrder.brand = skuData.get("brand")

                            flipkartOrder.model = skuData.get("model_name")

                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))

                            flipkartOrder.title =skuData.get("source_product_name")

                    session.commit()

                    nextUrl = resString['next']  

    def _saveToAffiliate(self, offers):

        collection = self.db.flipkartOrderAffiliateInfo1

        count=0

        for row in offers:

            if count==0:

                count += 1

                continue

            offer = self.covertToObj(row)

            collection.save(offer)

    def covertToObj(self,offer):

        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer

        saleMap = {

               "affiliateorderitemid":affiliateorderitemid,

                "title":title,

                "productid":productid,

                "category":category,

                "quantity":quantity,

                "saleAmount":sales,

                "price":price,

                "commissionrate":commissionrate,

                "payOut":tentativecommission,

                "conversionStatus":status,

                "saleDate":orderdate,

                "saleschannel":saleschannel,

                "customertype":customertype,

                "affextparam1":affextparam1,

                "_id":affiliateorderitemid

        }

        return saleMap

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp= {}

        try:

            br = getBrowserObject()

            url = ORDER_TRACK_URL + urlparse(orderSuccessUrl).query 

            response = br.open(url)

            page = ungzipResponse(response)

            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]

            print merchantOrderId

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            sections = soup.findAll("div", {"class":"ui-app-card-body"})

            sections.pop(1)

            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

            merchantOrder.orderTrackingUrl = url

            for data in sections:

                name = data.findAll("span")

                i=0

                while i< len(name):

                    if "key" in str(name[i]):

                        if "Grand Total" in str(name[i]):

                            #Total Amount

                            merchantOrder.paidAmount = int(re.findall(r'\d+', name[i+1].text)[0])

                        elif "Order Date" in str(name[i]):

                            merchantOrder.placedOn = name[i+1].text 

                    i=i+1    

            merchantOrder.merchantOrderId = merchantOrderId

            mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

            fkSubOrders = mainOrder.findAll("li")

            #remove unwanted list

            fkSubOrders.pop(-1)

            subOrders = []

            merchantOrder.subOrders = subOrders

            #fetching suborders details

            for subOrder in fkSubOrders:

                y = subOrder.find("header").findAll("span")

                for suborderId in y:

                    if "value emp" in str(suborderId):

                        merchantSubOrderId = suborderId.text

                        break

                orderItems = subOrder.findAll("div", "product-list-item", recursive=True)

                i=0

                for orderItem in orderItems:

                    merchantOrder.placedOn

                    merchantSubOrderId = merchantSubOrderId + str(i)

                    divs = orderItem.findAll('div', recursive=False)

                    content = divs[0]

                    orderTracking = divs[2]

                    orderTrackingDet = divs[3]

                    imgUrl = content.find('img')['src']

                    lineDet = content.find("div",{"class":"product-info"})

                    productTitle = lineDet.find("a",{"class":"product-title"}).text

                    productUrl =  str(lineDet.find("a")['href'])

                    start='pid='

                    s=str(lineDet.find("a")['href'])

                    productCode = re.findall(re.escape(start)+"(.*)",s)[0].strip()

                    mname = lineDet.findAll("span")

                    k=0

                    while k<len(mname):

                        if "note" in str(mname[k]):

                            if "Color:" in str(mname[k]):

                                productTitle = productTitle + " " + mname[k+1].text

                            elif "Qty:" in str(mname[k]):

                                quantity = int(mname[k+1].text)

                            elif "Subtotal:" in str(mname[k]):

                                amountPaid = int(re.findall(r'\d+', mname[k+1].text)[0])

                            elif "Delivery:" in str(mname[k]):

                                #Delivery Date for sub order

                                print "Delivery Date " +mname[k+1].text

                        k=k+1

                    merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)

                    merchantsubOrder.imgUrl  = imgUrl

                    merchantsubOrder.productCode = productCode

                    merchantsubOrder.merchantSubOrderId = merchantSubOrderId

                    print "productCode", productCode

                    print "amountPaid", amountPaid

                    cashbackAmount, cashbackPercent = self.getCashbackAmount(productCode, amountPaid)

                    cashbackStatus = Store.CB_PENDING

                    if cashbackAmount <= 0:

                        cashbackStatus = Store.CB_NA

                    merchantsubOrder.cashBackAmount = cashbackAmount

                    merchantsubOrder.cashBackStatus = cashbackStatus      

                    merchantsubOrder.cashBackPercentage = cashbackPercent

                    merchantsubOrder.detailedStatus = str(orderItem.find("div", "tracking-remark", recursive=True).text)   

                    subOrders.append(merchantsubOrder)

                    #To track shipping details         

                    status=ORDER_PLACED    

                    tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                    if "approveDetails-complete" in str(tr):

                        if "processingDetails-complete" in str(tr):

                            if "shippingDetails-complete" in str(tr):

                                if "delivery-complete" in str(tr):

                                    status = MStore.ORDER_DELIVERED

                                else:     

                                    status = MStore.ORDER_SHIPPED

                                    orderTracking.find('div', {'class':'tracking-remark active'})

                            else:

                                status = MStore.ORDER_PLACED

                        else: 

                            status = MStore.ORDER_PLACED

                    elif "approveDetails-ongoing" in str(tr):

                        status=MStore.ORDER_PLACED             

                    if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):

                        status = MStore.ORDER_CANCELLED    

                    print "Sub Order Status " + str(status)

                    subOrder.status=status

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp 

        except:

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            return resp 

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)           

def _saveToOrderFlipkart(self, order):

        collection = self.db.merchantOrder

        order = collection.insert(order)

def hex_md5(password):

    m = hashlib.md5()

    print(m.digest())

def main():

    store = getStore(2)

    store.parseOrderRawHtml(1, '123', 14, '', 'https://www.flipkart.com/rv/orderConfirmation/orderresponse?reference_id=OD5071037231594670&token=06919f2c830cb1a5ebb9d2798f602261')

if __name__ == '__main__':

        main()