Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo

from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \

    Store as MStore, sourceMap, tprint

from dtr.storage.DataService import Clicks, Users, FlipkartOrders, OrdersRaw

from dtr.utils import utils

from dtr.utils.utils import fetchResponseUsingProxy, todict, getSkuData,\

    ORDER_PLACED

from elixir import *

from pprint import pprint

from pymongo.mongo_client import MongoClient

import StringIO

import csv

import hashlib

import importlib

import json

import mechanize

import pymongo

import re

import traceback

import urllib

import urllib2

from urlparse import urlparse

import zipfile

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

ORDER_TRACK_URL='https://www.flipkart.com/order_details?src=or&pr=1&type=physical&'

AFFILIATE_URL='https://affiliate.flipkart.com/'

AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'

AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"

AFF_STATUS_CANCELLED='cancelled'

AFF_STATUS_APPROVED='approved'

AFF_STATUS_DISAPPROVED='disapproved'

AFF_STATUS_PENDING='pending'

statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]

categoryMap = {3:"Mobiles", 5:"Tablets"}

mobileAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4'

headers = { 

            'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',

            'Connection':'keep-alive',

            'Accept-Encoding' : 'gzip,deflate,sdch'

        }

TRACK_URLS = []

TRACK_HEADERS = {

        "sn":"2.VIBAC8BD9D21024F649B0482FAC385814A.SIE6C5F25D78974A5EB850D89167D52009.VSE7C76F85C2EE463786F89407CA46CA31.1492756338",

        "Connection":"keep-alive",

        "Referer":"https://www.flipkart.com/rv/orders?link=myAccounts",

        "cache-control":"no-cache",

        "Accept":"/",

        "Accept-Language":"en-US,en;q=0.8",

        "content-type":"application/json",

        "User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36",

        "flipkart_secure":"true",

        "sc":"",

        "Accept-Encoding":"gzip:deflate:sdch:br",

        "x-user-agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36 FKUA/msite/0.0.1/msite/Mobile,Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36 FKUA/msite/0.0.1/msite/Mobile",

        "pragma":"no-cache",

        "issecureresource":"true"

}

class Store(MStore):

    OrderStatusMap = {

                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping'],

                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],

                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered'],

                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.']

                      }

    def __init__(self,store_id):

        client = MongoClient('mongodb://localhost:27017/')

        self.db = client.dtr

        super(Store, self).__init__(store_id)

    def getName(self):

        return "flipkart"

    def trackOrdersForUser(self, userId, url, rawHtml):

        rawHtml = re.sub(r'[^\x00-\x7F]+',' ', rawHtml)

        if url not in TRACK_URLS:

            for lis in re.findall('SN:"(.*?)"', rawHtml):

                TRACK_HEADERS['sn'] = lis

                break

            for lis in re.findall('SN:"(.*?)"', rawHtml):

                TRACK_HEADERS['User-Agent'] = lis.split(" fkUA")[0]

                TRACK_HEADERS['x-user-agent'] = lis

                break

            return {"headers":TRACK_HEADERS, "urls":TRACK_URLS}

        else:

            return 'PARSED_SUCCESS'

            #parse json here

            #jsonObject = json.loads(readSSh("/home/amit/node/serve/res.json"))

    def requestDownload(self):

    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"

        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

        br = getBrowserObject()

        br.set_debug_responses(True)

        br.open(AFFILIATE_URL)

        response = br.response()  # copy

        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]

        data = {

                    'j_username':'saholic1@gmail.com',

                    'j_password':'spice@2020'

            }

        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))

        till = datetime.strftime(date.today(),"%Y-%m-%d")

        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")

        #target = open("dowloadreportids", 'w')

        #target.truncate()

        for status in statuses:

            #try:

                data = {"till":till, "from":start, "filter":status}

                print json.dumps(data)

                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))

                print request

                response = utils.ungzipResponse(br.open(request))

                response = json.loads(response)

                print response

                if(response['status']=="OK"):

                    self.db.flipkartdownloadids.save(response)

                else:

                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")

                    #break 

            #except:

                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")

                #break 

    def scrapeStoreOrders(self,):

        orders = self._getActiveOrders()

        for order in orders:

            print "Order", self.store_name, order['orderId']

            try:

                closed = True

                url = ORDER_TRACK_URL + urlparse(order['orderSuccessUrl']).query    

                page = fetchResponseUsingProxy(url, headers)

                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

                sections = soup.findAll("div", {"class":"ui-app-card-body"})

                sections.pop(1)

                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

                fkSubOrders = mainOrder.findAll("li")

                #remove unwanted list

                fkSubOrders.pop(-1)

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                #fetching suborders details

                for subOrder in fkSubOrders:

                    updateMap = {}

                    ul = subOrder.find("ul")

                    if ul is None:

                        ul = subOrder.findAll("div", recursive=False)[0].div.div

                    orderItems = ul.findAll("div", recursive=False)

                    for orderItem in orderItems:

                        closedStatus = False

                        divs = orderItem.findAll('div', recursive=False)

                        orderTracking = divs[2]

                        merchantSubOrderId = divs[3].get('id')

                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

                        if subOrder is None:

                            break

                        elif subOrder['closed']:

                            break

                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})

                        cashbackStatus = subOrder.get("cashBackStatus")

                        if orderTrackingDetDiv is not None:

                            orderTrackingDet = str(orderTrackingDetDiv.find("div", "tracking-remark").text)

                            updateMap["subOrders.$.detailedStatus"] = orderTrackingDet

                            tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                            tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                            if "approveDetails-ongoing" in str(tr) or "processingDetails-ongoing" in str(tr):

                                status=MStore.ORDER_PLACED

                            elif "shippingDetails-ongoing" in str(tr):

                                status = MStore.ORDER_SHIPPED

                            elif "delivery-complete" in str(tr):

                                status = MStore.ORDER_DELIVERED

                                if cashbackStatus != Store.CB_NA:

                                    cashbackStatus = Store.CB_APPROVED

                                closedStatus = True

                            if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):

                                status = MStore.ORDER_CANCELLED

                                closedStatus = True

                                if cashbackStatus == Store.CB_PENDING:

                                    cashbackStatus = Store.CB_CANCELLED

                        else:

                            updateMap["subOrders.$.detailedStatus"] = "Refunded"

                            status = MStore.ORDER_CANCELLED

                            closedStatus = True

                            if cashbackStatus == Store.CB_PENDING:

                                cashbackStatus = Store.CB_CANCELLED

                        print "Sub Order Status " + str(status)

                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus

                        updateMap["subOrders.$.status"] = status

                        updateMap["subOrders.$.closed"] = closedStatus

                        if closed:

                            closed = closedStatus

                        bulk.find(findMap).update({'$set' : updateMap})

                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})

                bulk.execute()            

            except:

                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})

                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())

                traceback.print_exc()

    def scrapeAffiliate(self, deltaDays=0):

        if deltaDays is None:

            deltaDays=0

        endDate=date.today() - timedelta(days=1)

        startDate = endDate - timedelta(days=deltaDays)

        endDate = datetime.strftime(endDate, "%Y-%m-%d")

        startDate = datetime.strftime(startDate, "%Y-%m-%d")

        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=0"

        for status in statuses:

            nextUrl = url%(startDate, endDate, status)

            while nextUrl:

                req = urllib2.Request(nextUrl)

                nextUrl=''

                req.add_header('Fk-Affiliate-Id', 'saholic1g')

                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')

                resp = urllib2.urlopen(req)

                resString = json.loads(resp.read())

                orderList = resString["orderList"]

                if orderList:

                    for order in orderList:

                        order['sales'] = int(order['sales']['amount'])

                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])

                        subTagId = order.get("affExtParam1")

                        userId = None

                        email = None

                        if subTagId:

                            click = session.query(Clicks).filter_by(tag = subTagId).first()

                            if click is not None:

                                userId= click.user_id 

                                user = session.query(Users.email).filter_by(id = userId).first()

                                if user is not None:

                                    email = user.email

                        flipkartOrder = FlipkartOrders()

                        flipkartOrder.subtagId = subTagId

                        flipkartOrder.user_id = userId

                        flipkartOrder.identifier = order.get("identifier")

                        flipkartOrder.email = email

                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%d-%m-%Y %H:%M:%S")

                        flipkartOrder.status = order.get("status")

                        flipkartOrder.title = order.get("title")

                        flipkartOrder.price = order.get("price")

                        flipkartOrder.quantity = order.get("quantity")

                        flipkartOrder.productCode = order.get("productId")

                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")

                        flipkartOrder.payOut = order['tentativeCommission']

                        flipkartOrder.payOutPercentage = order['commissionRate']

                        skuData = getSkuData(2, order.get("productId"))

                        if skuData is not None:

                            flipkartOrder.catalogId = skuData.get("skuBundleId")

                            flipkartOrder.brand = skuData.get("brand")

                            flipkartOrder.model = skuData.get("model_name")

                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))

                            flipkartOrder.title =skuData.get("source_product_name")

                    session.commit()

                    nextUrl = resString['next']  

    def _saveToAffiliate(self, offers):

        collection = self.db.flipkartOrderAffiliateInfo1

        count=0

        for row in offers:

            if count==0:

                count += 1

                continue

            offer = self.covertToObj(row)

            collection.save(offer)

    def covertToObj(self,offer):

        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer

        saleMap = {

               "affiliateorderitemid":affiliateorderitemid,

                "title":title,

                "productid":productid,

                "category":category,

                "quantity":quantity,

                "saleAmount":sales,

                "price":price,

                "commissionrate":commissionrate,

                "payOut":tentativecommission,

                "conversionStatus":status,

                "saleDate":orderdate,

                "saleschannel":saleschannel,

                "customertype":customertype,

                "affextparam1":affextparam1,

                "_id":affiliateorderitemid

        }

        return saleMap

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp= {}

        try:

            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]

            if len(self.db.merchantOrder.find({"merchantOrderId":merchantOrderId})) == 0:

                br = getBrowserObject(mobileAgent)

                url = ORDER_TRACK_URL + urlparse(orderSuccessUrl).query 

                response = br.open(url)

                page = ungzipResponse(response)

                print merchantOrderId

                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

                sections = soup.findAll("div", {"class":"ui-app-card-body"})

                sections.pop(1)

                merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

                merchantOrder.closed = True

                merchantOrder.orderTrackingUrl = url

                for data in sections:

                    name = data.findAll("span")

                    i=0

                    while i< len(name):

                        if "key" in str(name[i]):

                            if "Grand Total" in str(name[i]):

                                #Total Amount

                                merchantOrder.paidAmount = int(re.findall(r'\d+', name[i+1].text)[0])

                            elif "Order Date" in str(name[i]):

                                merchantOrder.placedOn = name[i+1].text 

                        i=i+1    

                merchantOrder.merchantOrderId = merchantOrderId

                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

                fkSubOrders = mainOrder.findAll("li")

                #remove unwanted list

                fkSubOrders.pop(-1)

                subOrders = []

                merchantOrder.subOrders = subOrders

                for subOrder in fkSubOrders:

                    orderItems = subOrder.findAll("div", "product-list-item", recursive=True)

                    for orderItem in orderItems:

                        if not orderItem.has_key("id"):

                            continue

                        divs = orderItem.findAll('div', recursive=False)

                        content = divs[0]

                        orderTracking = divs[2]

                        merchantSubOrderId = divs[3].get('id')

                        imgUrl = content.find('img')['src']

                        lineDet = content.find("div",{"class":"product-info"})

                        productTitle = lineDet.find("a",{"class":"product-title"}).text

                        productUrl =  str(lineDet.find("a")['href'])

                        start='pid='

                        s=str(lineDet.find("a")['href'])

                        productCode = re.findall(re.escape(start)+"(.*)",s)[0].strip()

                        mname = lineDet.findAll("span")

                        k=0

                        while k<len(mname):

                            if "note" in str(mname[k]):

                                if "Color:" in str(mname[k]):

                                    productTitle = productTitle + " " + mname[k+1].text

                                elif "Qty:" in str(mname[k]):

                                    quantity = int(mname[k+1].text)

                                elif "Subtotal:" in str(mname[k]):

                                    amountPaid = int(re.findall(r'\d+', mname[k+1].text)[0])

                                elif "Delivery:" in str(mname[k]):

                                    #Delivery Date for sub order

                                    print "Delivery Date " +mname[k+1].text

                            k=k+1

                        merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)

                        merchantsubOrder.imgUrl  = imgUrl

                        merchantsubOrder.productCode = productCode

                        merchantsubOrder.merchantSubOrderId = merchantSubOrderId

                        print "productCode", productCode

                        print "amountPaid", amountPaid

                        cashbackAmount, cashbackPercent = self.getCashbackAmount(productCode, amountPaid)

                        cashbackStatus = Store.CB_PENDING

                        if cashbackAmount <= 0:

                            cashbackStatus = Store.CB_NA

                        merchantsubOrder.cashBackAmount = cashbackAmount

                        merchantsubOrder.cashBackStatus = cashbackStatus      

                        merchantsubOrder.cashBackPercentage = cashbackPercent

                        activePart = orderItem.find("div", "c-tabs-content m-top active")

                        if activePart is not None:

                            merchantsubOrder.detailedStatus = str(activePart.find("div", "tracking-remark").text)   

                        #To track shipping details         

                        status=ORDER_PLACED

                        tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                        if "approveDetails-ongoing" in str(tr) or "processingDetails-ongoing" in str(tr):

                            pass

                        elif "shippingDetails-ongoing" in str(tr):

                            status = MStore.ORDER_SHIPPED

                        elif "delivery-complete" in str(tr):

                            status = MStore.ORDER_DELIVERED

                            if cashbackStatus!=MStore.CB_NA:

                                merchantsubOrder.cashBackStatus = MStore.CB_APPROVED

                            merchantsubOrder.closed = True

                        if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):

                            status = MStore.ORDER_CANCELLED

                            if merchantsubOrder.cashBackStatus != MStore.CB_NA:    

                                merchantsubOrder.cashBackStatus = MStore.CB_CANCELLED

                            merchantsubOrder.closed = True

                        print "Sub Order Status " + str(status)

                        merchantsubOrder.status=status

                        if merchantOrder.closed:

                            merchantOrder.closed = merchantsubOrder.closed 

                        subOrders.append(merchantsubOrder)

                if self._saveToOrder(todict(merchantOrder)):

                    resp['result'] = 'ORDER_CREATED'

                else:

                    resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp 

        except:

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            return resp 

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)           

def _saveToOrderFlipkart(self, order):

        collection = self.db.merchantOrder

        order = collection.insert(order)

def hex_md5(password):

    m = hashlib.md5()

    print(m.digest())

def main():

    store = getStore(2)

    store.parseOrderRawHtml(1, '1234', 14, '', 'https://www.flipkart.com/rv/orderConfirmation/orderresponse?pr=1&reference_id=OD1088257258903640&src=or&type=physical&token=e0fa1db468ba77bfb8cdac938e051ef3')

    #store.scrapeStoreOrders()

if __name__ == '__main__':

        main()