Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo

from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \

    Store as MStore, sourceMap, tprint

from dtr.storage.DataService import Clicks, Users, FlipkartOrders

from dtr.utils import utils

from dtr.utils.utils import fetchResponseUsingProxy, todict, getSkuData,\

    ORDER_PLACED

from elixir import *

from pprint import pprint

from pymongo.mongo_client import MongoClient

import StringIO

import csv

import hashlib

import importlib

import json

import mechanize

import pymongo

import re

import traceback

import urllib

import urllib2

from urlparse import urlparse

import zipfile

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

ORDER_TRACK_URL='https://m.flipkart.com/order_details?src=or&pr=1&type=physical&'

AFFILIATE_URL='https://affiliate.flipkart.com/'

AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'

AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"

AFF_STATUS_CANCELLED='cancelled'

AFF_STATUS_APPROVED='approved'

AFF_STATUS_DISAPPROVED='disapproved'

AFF_STATUS_PENDING='pending'

statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]

categoryMap = {3:"Mobiles", 5:"Tablets"}

class Store(MStore):

    OrderStatusMap = {

                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping'],

                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],

                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered'],

                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.']

                      }

    def __init__(self,store_id):

        client = MongoClient('mongodb://localhost:27017/')

        self.db = client.dtr

        super(Store, self).__init__(store_id)

    def getName(self):

        return "flipkart"

    def requestDownload(self):

    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"

        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

        br = getBrowserObject()

        br.set_debug_responses(True)

        br.open(AFFILIATE_URL)

        response = br.response()  # copy

        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]

        data = {

                    'j_username':'saholic1@gmail.com',

                    'j_password':'spice@2020'

            }

        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))

        till = datetime.strftime(date.today(),"%Y-%m-%d")

        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")

        #target = open("dowloadreportids", 'w')

        #target.truncate()

        for status in statuses:

            #try:

                data = {"till":till, "from":start, "filter":status}

                print json.dumps(data)

                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))

                print request

                response = utils.ungzipResponse(br.open(request))

                response = json.loads(response)

                print response

                if(response['status']=="OK"):

                    self.db.flipkartdownloadids.save(response)

                else:

                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")

                    #break 

            #except:

                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")

                #break 

    def scrapeStoreOrders(self,):

        orders = self._getActiveOrders()

        for order in orders:

            print "Order", self.store_name, order['orderId']

            try:

                closed = True

                url = ORDER_TRACK_URL + urlparse(order['orderSuccessUrl']).query

                page = fetchResponseUsingProxy(url)

                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

                sections = soup.findAll("div", {"class":"ui-app-card-body"})

                sections.pop(1)

                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

                fkSubOrders = mainOrder.findAll("li")

                #remove unwanted list

                fkSubOrders.pop(-1)

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                #fetching suborders details

                for subOrder in fkSubOrders:

                    updateMap = {}

                    y = subOrder.find("header").findAll("span")

                    for suborderId in y:

                        if "value emp" in str(suborderId):

                            merchantSubOrderId = suborderId.text

                            break

                    ul = subOrder.find("ul")

                    if ul is None:

                        ul = subOrder.findAll("div", recursive=False)[0].div.div

                    orderItems = ul.findAll("div", recursive=False)

                    i=0

                    for orderItem in orderItems:

                        closedStatus = False

                        merchantSubOrderId = merchantSubOrderId + str(i)

                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

                        if subOrder is None:

                            break

                        elif subOrder['closed']:

                            break

                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                        divs = orderItem.findAll('div', recursive=False)

                        orderTracking = divs[2]

                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})

                        orderTrackingDet = str(orderTrackingDetDiv.find("div", "tracking-remark").text)

                        updateMap["subOrders.$.detailedStatus"] = orderTrackingDet

                        '''status = self._getStatusFromDetailedStatus(orderTrackingDet)'''

                        tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                        cashbackStatus = subOrder.get("cashBackStatus")

                        tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                        if "approveDetails-ongoing" in str(tr) or "processingDetails-ongoing" in str(tr):

                            status=MStore.ORDER_PLACED

                        elif "shippingDetails-ongoing" in str(tr):

                            status = MStore.ORDER_SHIPPED

                        elif "delivery-complete" in str(tr):

                            status = MStore.ORDER_DELIVERED

                            if cashbackStatus == Store.CB_PENDING:

                                cashbackStatus = Store.CB_APPROVED

                            closedStatus = True

                        if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):

                            status = MStore.ORDER_CANCELLED

                            closedStatus = True

                            if cashbackStatus == Store.CB_PENDING:

                                cashbackStatus = Store.CB_CANCELLED

                        print "Sub Order Status " + str(status)

                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus

                        updateMap["subOrders.$.status"] = status

                        updateMap["subOrders.$.closed"] = closedStatus

                        if closed:

                            closed = closedStatus

                        bulk.find(findMap).update({'$set' : updateMap})

                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})

                bulk.execute()            

            except:

                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})

                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())

                traceback.print_exc()

    def scrapeAffiliate(self, deltaDays=0):

        if deltaDays is None:

            deltaDays=0

        endDate=date.today() - timedelta(days=1)

        startDate = endDate - timedelta(days=deltaDays)

        endDate = datetime.strftime(endDate, "%Y-%m-%d")

        startDate = datetime.strftime(startDate, "%Y-%m-%d")

        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=0"

        for status in statuses:

            nextUrl = url%(startDate, endDate, status)

            while nextUrl:

                req = urllib2.Request(nextUrl)

                nextUrl=''

                req.add_header('Fk-Affiliate-Id', 'saholic1g')

                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')

                resp = urllib2.urlopen(req)

                resString = json.loads(resp.read())

                orderList = resString["orderList"]

                if orderList:

                    for order in orderList:

                        order['sales'] = int(order['sales']['amount'])

                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])

                        subTagId = order.get("affExtParam1")

                        userId = None

                        email = None

                        if subTagId:

                            click = session.query(Clicks).filter_by(tag = subTagId).first()

                            if click is not None:

                                userId= click.user_id 

                                user = session.query(Users.email).filter_by(id = userId).first()

                                if user is not None:

                                    email = user.email

                        flipkartOrder = FlipkartOrders()

                        flipkartOrder.subtagId = subTagId

                        flipkartOrder.user_id = userId

                        flipkartOrder.identifier = order.get("identifier")

                        flipkartOrder.email = email

                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%d-%m-%Y %H:%M:%S")

                        flipkartOrder.status = order.get("status")

                        flipkartOrder.title = order.get("title")

                        flipkartOrder.price = order.get("price")

                        flipkartOrder.quantity = order.get("quantity")

                        flipkartOrder.productCode = order.get("productId")

                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")

                        flipkartOrder.payOut = order['tentativeCommission']

                        flipkartOrder.payOutPercentage = order['commissionRate']

                        skuData = getSkuData(2, order.get("productId"))

                        if skuData is not None:

                            flipkartOrder.catalogId = skuData.get("skuBundleId")

                            flipkartOrder.brand = skuData.get("brand")

                            flipkartOrder.model = skuData.get("model_name")

                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))

                            flipkartOrder.title =skuData.get("source_product_name")

                    session.commit()

                    nextUrl = resString['next']  

    def _saveToAffiliate(self, offers):

        collection = self.db.flipkartOrderAffiliateInfo1

        count=0

        for row in offers:

            if count==0:

                count += 1

                continue

            offer = self.covertToObj(row)

            collection.save(offer)

    def covertToObj(self,offer):

        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer

        saleMap = {

               "affiliateorderitemid":affiliateorderitemid,

                "title":title,

                "productid":productid,

                "category":category,

                "quantity":quantity,

                "saleAmount":sales,

                "price":price,

                "commissionrate":commissionrate,

                "payOut":tentativecommission,

                "conversionStatus":status,

                "saleDate":orderdate,

                "saleschannel":saleschannel,

                "customertype":customertype,

                "affextparam1":affextparam1,

                "_id":affiliateorderitemid

        }

        return saleMap

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp= {}

        try:

            br = getBrowserObject()

            url = ORDER_TRACK_URL + urlparse(orderSuccessUrl).query 

            response = br.open(url)

            page = ungzipResponse(response)

            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]

            print merchantOrderId

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            sections = soup.findAll("div", {"class":"ui-app-card-body"})

            sections.pop(1)

            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

            merchantOrder.closed = True

            merchantOrder.orderTrackingUrl = url

            for data in sections:

                name = data.findAll("span")

                i=0

                while i< len(name):

                    if "key" in str(name[i]):

                        if "Grand Total" in str(name[i]):

                            #Total Amount

                            merchantOrder.paidAmount = int(re.findall(r'\d+', name[i+1].text)[0])

                        elif "Order Date" in str(name[i]):

                            merchantOrder.placedOn = name[i+1].text 

                    i=i+1    

            merchantOrder.merchantOrderId = merchantOrderId

            mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

            fkSubOrders = mainOrder.findAll("li")

            #remove unwanted list

            fkSubOrders.pop(-1)

            subOrders = []

            merchantOrder.subOrders = subOrders

            #fetching suborders details

            for subOrder in fkSubOrders:

                y = subOrder.find("header").findAll("span")

                for suborderId in y:

                    if "value emp" in str(suborderId):

                        merchantSubOrderId = suborderId.text

                        break

                orderItems = subOrder.findAll("div", "product-list-item", recursive=True)

                i=0

                for orderItem in orderItems:

                    merchantOrder.placedOn

                    merchantSubOrderId = merchantSubOrderId + str(i)

                    divs = orderItem.findAll('div', recursive=False)

                    content = divs[0]

                    orderTracking = divs[2]

                    orderTrackingDet = divs[3]

                    imgUrl = content.find('img')['src']

                    lineDet = content.find("div",{"class":"product-info"})

                    productTitle = lineDet.find("a",{"class":"product-title"}).text

                    productUrl =  str(lineDet.find("a")['href'])

                    start='pid='

                    s=str(lineDet.find("a")['href'])

                    productCode = re.findall(re.escape(start)+"(.*)",s)[0].strip()

                    mname = lineDet.findAll("span")

                    k=0

                    while k<len(mname):

                        if "note" in str(mname[k]):

                            if "Color:" in str(mname[k]):

                                productTitle = productTitle + " " + mname[k+1].text

                            elif "Qty:" in str(mname[k]):

                                quantity = int(mname[k+1].text)

                            elif "Subtotal:" in str(mname[k]):

                                amountPaid = int(re.findall(r'\d+', mname[k+1].text)[0])

                            elif "Delivery:" in str(mname[k]):

                                #Delivery Date for sub order

                                print "Delivery Date " +mname[k+1].text

                        k=k+1

                    merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)

                    merchantsubOrder.imgUrl  = imgUrl

                    merchantsubOrder.productCode = productCode

                    merchantsubOrder.merchantSubOrderId = merchantSubOrderId

                    print "productCode", productCode

                    print "amountPaid", amountPaid

                    cashbackAmount, cashbackPercent = self.getCashbackAmount(productCode, amountPaid)

                    cashbackStatus = Store.CB_PENDING

                    if cashbackAmount <= 0:

                        cashbackStatus = Store.CB_NA

                    merchantsubOrder.cashBackAmount = cashbackAmount

                    merchantsubOrder.cashBackStatus = cashbackStatus      

                    merchantsubOrder.cashBackPercentage = cashbackPercent

                    merchantsubOrder.detailedStatus = str(orderItem.find("div", "c-tabs-content m-top active").find("div", "tracking-remark").text)   

                    #To track shipping details         

                    status=ORDER_PLACED

                    tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                    if "approveDetails-ongoing" in str(tr) or "processingDetails-ongoing" in str(tr):

                        pass

                    elif "shippingDetails-ongoing" in str(tr):

                        status = MStore.ORDER_SHIPPED

                    elif "delivery-complete" in str(tr):

                        status = MStore.ORDER_DELIVERED

                        merchantsubOrder.cashBackStatus = MStore.CB_APPROVED

                        merchantsubOrder.closed = True

                    if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):

                        status = MStore.ORDER_CANCELLED    

                        merchantsubOrder.cashBackStatus = MStore.CB_CANCELLED

                        merchantsubOrder.closed = True

                    print "Sub Order Status " + str(status)

                    merchantsubOrder.status=status

                    if merchantOrder.closed:

                        merchantOrder.closed = merchantsubOrder.closed 

                    subOrders.append(merchantsubOrder)

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp 

        except:

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            return resp 

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)           

def _saveToOrderFlipkart(self, order):

        collection = self.db.merchantOrder

        order = collection.insert(order)

def hex_md5(password):

    m = hashlib.md5()

    print(m.digest())

def main():

    store = getStore(2)

    #store.parseOrderRawHtml(1, '123', 14, '', 'https://www.flipkart.com/rv/orderConfirmation/orderresponse?reference_id=OD4072085355271530&token=c8a726cb1495232e00338fb0eecb4f40')

    store.scrapeStoreOrders()

if __name__ == '__main__':

        main()