Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo

from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \

    Store as MStore, sourceMap, tprint

from dtr.reports.affiliatereco import getSkuData

from dtr.storage.DataService import Clicks, Users, FlipkartOrders

from dtr.utils import utils

from dtr.utils.utils import fetchResponseUsingProxy

from elixir import *

from pprint import pprint

from pymongo.mongo_client import MongoClient

import StringIO

import csv

import hashlib

import importlib

import json

import mechanize

import pymongo

import re

import traceback

import urllib

import zipfile

import urllib2

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

ORDER_TRACK_URL='https://m.flipkart.com/order_details'

AFFILIATE_URL='https://affiliate.flipkart.com/'

AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'

AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"

AFF_STATUS_CANCELLED='cancelled'

AFF_STATUS_APPROVED='approved'

AFF_STATUS_DISAPPROVED='disapproved'

AFF_STATUS_PENDING='pending'

statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]

categoryMap = {3:"Mobiles", 5:"Tablets"}

class Store(MStore):

    OrderStatusMap = {

                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping'],

                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],

                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered'],

                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.']

                      }

    def __init__(self,store_id):

        client = MongoClient('mongodb://localhost:27017/')

        self.db = client.dtr

        super(Store, self).__init__(store_id)

    def getName(self):

        return "flipkart"

    def requestDownload(self):

    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"

        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

        br = getBrowserObject()

        br.set_debug_responses(True)

        br.open(AFFILIATE_URL)

        response = br.response()  # copy

        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]

        data = {

                    'j_username':'saholic1@gmail.com',

                    'j_password':'spice@2020'

            }

        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))

        till = datetime.strftime(date.today(),"%Y-%m-%d")

        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")

        #target = open("dowloadreportids", 'w')

        #target.truncate()

        for status in statuses:

            #try:

                data = {"till":till, "from":start, "filter":status}

                print json.dumps(data)

                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))

                print request

                response = utils.ungzipResponse(br.open(request))

                response = json.loads(response)

                print response

                if(response['status']=="OK"):

                    self.db.flipkartdownloadids.save(response)

                else:

                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")

                    #break 

            #except:

                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")

                #break 

    def scrapeStoreOrders(self,):

        orders = self._getActiveOrders()

        for order in orders:

            print "Order", self.store_name, order['orderId']

            try:

                closed = True

                url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]

                page = fetchResponseUsingProxy(url)

                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

                sections = soup.findAll("div", {"class":"ui-app-card-body"})

                sections.pop(1)

                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

                fkSubOrders = mainOrder.findAll("li")

                #remove unwanted list

                fkSubOrders.pop(-1)

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                #fetching suborders details

                for subOrder in fkSubOrders:

                    updateMap = {}

                    y = subOrder.find("header").findAll("span")

                    for suborderId in y:

                        if "value emp" in str(suborderId):

                            merchantSubOrderId = suborderId.text

                            break

                    ul = subOrder.find("ul")

                    if ul is None:

                        ul = subOrder.findAll("div", recursive=False)[0].div.div

                    orderItems = ul.findAll("div", recursive=False)

                    i=0

                    for orderItem in orderItems:

                        closedStatus = False

                        merchantSubOrderId = merchantSubOrderId + str(i)

                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

                        if subOrder is None:

                            break

                        elif subOrder['closed']:

                            break

                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                        divs = orderItem.findAll('div', recursive=False)

                        orderTracking = divs[2]

                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})

                        orderTrackingDet = orderTrackingDetDiv.find('div',{'class':re.compile('tracking-remark')}).text

                        updateMap["subOrders.$.detailedStatus"] = orderTrackingDet

                        status = self._getStatusFromDetailedStatus(orderTrackingDet)

                        tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                        cashbackStatus = subOrder.get("cashBackStatus") 

                        if "approveDetails-complete" in str(tr):

                            if "processingDetails-complete" in str(tr):

                                if "shippingDetails-complete" in str(tr):

                                    if "delivery-complete" in str(tr):

                                        status = MStore.ORDER_DELIVERED

                                        if cashbackStatus == Store.CB_PENDING:

                                            cashbackStatus = Store.CB_APPROVED

                                        closedStatus = True

                                    else:     

                                        status = MStore.ORDER_SHIPPED

                                        courierTrackAnchor = orderTrackingDetDiv.find('a')

                                        trackingUrl = courierTrackAnchor("href")

                                        trackingText = courierTrackAnchor.text

                                        courierArr = trackingText.split(' : ')

                                        updateMap["subOrders.$.trackingUrl"] = trackingUrl

                                        updateMap["subOrders.$.trackingNumber"] = courierArr[1]

                                        updateMap["subOrders.$.courierName"] = courierArr[0]

                                else:

                                    status = MStore.ORDER_CANCELLED

                                    closedStatus = True

                                    if cashbackStatus == Store.CB_PENDING:

                                        cashbackStatus = Store.CB_CANCELLED

                            else: 

                                status = MStore.ORDER_PLACED

                        elif str(tr) in ["approveDetails-ongoing"]:

                            status=MStore.ORDER_PLACED             

                        if "dead" in str(tr) or status==MStore.ORDER_CANCELLED:

                            status = MStore.ORDER_CANCELLED

                            closedStatus = True

                            if cashbackStatus == Store.CB_PENDING:

                                cashbackStatus = Store.CB_CANCELLED

                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus

                        updateMap["subOrders.$.status"] = status

                        updateMap["subOrders.$.closed"] = closedStatus

                        if closed:

                            closed = closedStatus

                        bulk.find(findMap).update({'$set' : updateMap})

                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})

                bulk.execute()            

            except:

                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})

                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())

                traceback.print_exc()

    def scrapeAffiliate(self, deltaDays=0):

        if deltaDays is None:

            deltaDays=0

        endDate=date.today() - timedelta(days=1)

        startDate = endDate - timedelta(days=deltaDays)

        endDate = datetime.strftime(endDate, "%Y-%m-%d")

        startDate = datetime.strftime(startDate, "%Y-%m-%d")

        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=%s"

        sum = 0

        for status in statuses:

            offset=-1

            while True:

                offset += 1

                reqUlrl = url%(startDate, endDate, status, offset) 

                req = urllib2.Request(reqUlrl)

                req.add_header('Fk-Affiliate-Id', 'saholic1g')

                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')

                resp = urllib2.urlopen(req)

                resString = json.loads(resp.read())

                orderList = resString["orderList"]

                if orderList:

                    for order in orderList:

                        print order

                        order['sales'] = int(order['sales']['amount'])

                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])

                        if order['status']!='failed':

                             sum += order['tentativeCommission']

                        self.db.flipkartOrderAffiliateInfo1.save(order)

                        skuData = getSkuData(2, order.get("productCode"))

                        if skuData is not None:

                            order['catalogId'] = skuData.get("skuBundleId")

                            order['brand'] = skuData.get("brand")

                            order['model'] = skuData.get("model_name")

                            order['category'] = categoryMap.get(skuData.get("category_id"))

                        subTagId = order.get("affExtParam1")

                        userId = None

                        email = None

                        if subTagId:

                            click = session.query(Clicks).filter_by(tag = subTagId).first()

                            if click is not None:

                                userId= click.user_id 

                                user = session.query(Users.email).filter_by(id = userId).first()

                                if user is not None:

                                    email = user.email

                        flipkartOrder = FlipkartOrders()

                        flipkartOrder.subtagId = subTagId

                        flipkartOrder.user_id = userId

                        flipkartOrder.identifier = order.get("identifier")

                        flipkartOrder.email = email

                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%m-%d-%Y")

                        flipkartOrder.status = order.get("status")

                        flipkartOrder.title = order.get("title")

                        flipkartOrder.price = order.get("price")

                        flipkartOrder.quantity = order.get("quantity")

                        flipkartOrder.productCode = order.get("productId")

                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")

                        flipkartOrder.payOut = order['tentativeCommission']

                        flipkartOrder.payOutPercentage = order['commissionRate']

                        skuData = getSkuData(2, order.get("productId"))

                        if skuData is not None:

                            flipkartOrder.catalogId = skuData.get("skuBundleId")

                            flipkartOrder.brand = skuData.get("brand")

                            flipkartOrder.model = skuData.get("model_name")

                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))

                            flipkartOrder.title =skuData.get("source_product_name")

                    session.commit()

                else:

                    break

    def _saveToAffiliate(self, offers):

        collection = self.db.flipkartOrderAffiliateInfo1

        count=0

        for row in offers:

            if count==0:

                count += 1

                continue

            offer = self.covertToObj(row)

            collection.save(offer)

    def covertToObj(self,offer):

        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer

        saleMap = {

               "affiliateorderitemid":affiliateorderitemid,

                "title":title,

                "productid":productid,

                "category":category,

                "quantity":quantity,

                "saleAmount":sales,

                "price":price,

                "commissionrate":commissionrate,

                "payOut":tentativecommission,

                "conversionStatus":status,

                "saleDate":orderdate,

                "saleschannel":saleschannel,

                "customertype":customertype,

                "affextparam1":affextparam1,

                "_id":affiliateorderitemid

        }

        return saleMap

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp= {}

        try:

            br = getBrowserObject()

            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

            response = br.open(url)

            page = ungzipResponse(response)

            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]

            print merchantOrderId

            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

            sections = soup.findAll("div", {"class":"ui-app-card-body"})

            sections.pop(1)

            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

            merchantOrder.orderTrackingUrl = url

            for data in sections:

                name = data.findAll("span")

                i=0

                while i< len(name):

                    if "key" in str(name[i]):

                        if "Grand Total" in str(name[i]):

                            #Total Amount

                            merchantOrder.paidAmount = int(re.findall(r'\d+', name[i+1].text)[0])

                        elif "Order Date" in str(name[i]):

                            merchantOrder.placedOn = name[i+1].text 

                    i=i+1    

            merchantOrder.merchantOrderId = merchantOrderId

            mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

            fkSubOrders = mainOrder.findAll("li")

            #remove unwanted list

            fkSubOrders.pop(-1)

            subOrders = []

            merchantOrder.subOrders = subOrders

            #fetching suborders details

            for subOrder in fkSubOrders:

                y = subOrder.find("header").findAll("span")

                for suborderId in y:

                    if "value emp" in str(suborderId):

                        merchantSubOrderId = suborderId.text

                        break

                orderItems = subOrder.find("ul").findAll("div", recursive=False)

                i=0

                for orderItem in orderItems:

                    merchantOrder.placedOn

                    merchantSubOrderId = merchantSubOrderId + str(i)

                    divs = orderItem.findAll('div', recursive=False)

                    content = divs[0]

                    orderTracking = divs[2]

                    orderTrackingDet = divs[3]

                    imgUrl = content.find('img')['src']

                    lineDet = content.find("div",{"class":"product-info"})

                    productTitle = lineDet.find("a",{"class":"product-title"}).text

                    productUrl =  str(lineDet.find("a")['href'])

                    start='pid='

                    s=str(lineDet.find("a")['href'])

                    productCode = re.findall(re.escape(start)+"(.*)",s)[0].strip()

                    mname = lineDet.findAll("span")

                    k=0

                    while k<len(mname):

                        if "note" in str(mname[k]):

                            if "Color:" in str(mname[k]):

                                productTitle = productTitle + " " + mname[k+1].text

                            elif "Qty:" in str(mname[k]):

                                quantity = int(mname[k+1].text)

                            elif "Subtotal:" in str(mname[k]):

                                amountPaid = int(re.findall(r'\d+', mname[k+1].text)[0])

                            elif "Delivery:" in str(mname[k]):

                                #Delivery Date for sub order

                                print "Delivery Date " +mname[k+1].text

                        k=k+1

                    merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)

                    merchantsubOrder.imgUrl  = imgUrl

                    merchantsubOrder.productCode = productCode

                    merchantsubOrder.merchantSubOrderId = merchantSubOrderId

                    print "productCode", productCode

                    print "amountPaid", amountPaid

                    cashbackAmount, cashbackPercent = self.getCashbackAmount(productCode, amountPaid)

                    cashbackStatus = Store.CB_PENDING

                    if cashbackAmount <= 0:

                        cashbackStatus = Store.CB_NA

                    merchantsubOrder.cashBackAmount = cashbackAmount

                    merchantsubOrder.cashBackStatus = cashbackStatus      

                    merchantsubOrder.cashBackPercentage = cashbackPercent      

                    subOrders.append(merchantsubOrder)

                    #To track shipping details         

                    status=-1    

                    tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                    if "approveDetails-complete" in str(tr):

                        if "processingDetails-complete" in str(tr):

                            if "shippingDetails-complete" in str(tr):

                                if "delivery-complete" in str(tr):

                                    status = MStore.ORDER_DELIVERED

                                else:     

                                    status = MStore.ORDER_SHIPPED

                                    orderTracking.find('div', {'class':'tracking-remark active'})

                            else:

                                status = MStore.ORDER_PLACED

                        else: 

                            status = MStore.ORDER_PLACED

                    elif str(tr) in ["approveDetails-ongoing"]:

                        status=MStore.ORDER_PLACED             

                    if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):

                        status = MStore.ORDER_CANCELLED    

                    print "Sub Order Status " + str(status)                      

                    trackingDetailsActive = orderTrackingDet.find("div",{"class":"c-tabs-content m-top active"})

                    print "Sub order tracking description " + trackingDetailsActive.text

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp 

        except:

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            return resp 

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)

    def flipkartOrderTracking(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        br = getBrowserObject()

        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]

        response = br.open(url)

        page = ungzipResponse(response, br)

        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})

        #fetching suborders details

        for subOrder in mainOrder:

            subOrd = subOrder.findAll("li")

            subOrd.pop(len(subOrd)-1)

            for productInfo in subOrd:

                status=-1    

                #To track shipping details         

                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})

                for trackingStatus in n:

                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                    if "approveDetails-complete" in str(tr):

                        print "Here"

                        if "processingDetails-complete" in str(tr):

                            print "There"

                            if "shippingDetails-complete" in str(tr):

                                print "Share"

                                if "delivery-complete" in str(tr):

                                    print "Last"

                                    status = 3

                                else:     

                                    status = 2

                            else:

                                status = 1 

                        else: 

                            status = 0     

                    if "dead" in str(tr):

                        status = 4    

                print status                      

def _saveToOrderFlipkart(self, order):

        collection = self.db.merchantOrder

        order = collection.insert(order)

def hex_md5(password):

    m = hashlib.md5()

    print(m.digest())

def main():

    store = getStore(2)

    store.scrapeAffiliate(200)

def todict(obj, classkey=None):

    if isinstance(obj, dict):

        data = {}

        for (k, v) in obj.items():

            data[k] = todict(v, classkey)

        return data

    elif hasattr(obj, "_ast"):

        return todict(obj._ast())

    elif hasattr(obj, "__iter__"):

        return [todict(v, classkey) for v in obj]

    elif hasattr(obj, "__dict__"):

        data = dict([(key, todict(value, classkey)) 

            for key, value in obj.__dict__.iteritems() 

            if not callable(value) and not key.startswith('_')])

        if classkey is not None and hasattr(obj, "__class__"):

            data[classkey] = obj.__class__.__name__

        return data

    else:

        return obj

if __name__ == '__main__':

        main()