Subversion Repositories SmartDukaan

'''

Created on Jan 15, 2015

@author: amit

'''

from BeautifulSoup import BeautifulSoup

from datetime import datetime, date, timedelta

from dtr import main

from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo, obj

from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \

    Store as MStore, sourceMap, tprint

from dtr.storage.DataService import Clicks, Users, FlipkartOrders, OrdersRaw

from dtr.utils import utils

from dtr.utils.utils import fetchResponseUsingProxy, todict, getSkuData, \

    ORDER_PLACED, readSSh

from elixir import *

from pprint import pprint

from pymongo.mongo_client import MongoClient

from urlparse import urlparse

import StringIO

import csv

import hashlib

import importlib

import json

import mechanize

import pymongo

import re

import time

import traceback

import urllib

import urllib2

import zipfile

USERNAME='saholic1@gmail.com'

PASSWORD='spice@2020'

ORDER_TRACK_URL='https://www.flipkart.com/order_details?src=or&pr=1&type=physical&'

AFFILIATE_URL='https://affiliate.flipkart.com/'

AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'

AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"

AFF_STATUS_CANCELLED='cancelled'

AFF_STATUS_APPROVED='approved'

AFF_STATUS_DISAPPROVED='disapproved'

AFF_STATUS_PENDING='pending'

statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]

categoryMap = {3:"Mobiles", 5:"Tablets"}

mobileAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4'

headers = { 

            'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4',

            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      

            'Accept-Language' : 'en-US,en;q=0.8',                     

            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',

            'Connection':'keep-alive',

            'Accept-Encoding' : 'gzip,deflate,sdch'

        }

TRACK_URLS = ['https://www.flipkart.com/api/1/self-serve/orders?pageNum=1&link=myAccounts', 'https://www.flipkart.com/api/1/self-serve/orders?pageNum=2&link=myAccounts', 'https://www.flipkart.com/api/1/self-serve/orders?pageNum=3&link=myAccounts']

TRACK_HEADERS = {

        "sn":"2.VIBAC8BD9D21024F649B0482FAC385814A.SIE6C5F25D78974A5EB850D89167D52009.VSE7C76F85C2EE463786F89407CA46CA31.1492756338",

        "Connection":"keep-alive",

        "Referer":"https://www.flipkart.com/rv/orders?link=myAccounts",

        "cache-control":"no-cache",

        "Accept":"/",

        "Accept-Language":"en-US,en;q=0.8",

        "content-type":"application/json",

        "User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36",

        "flipkart_secure":"true",

        "sc":"",

        "Accept-Encoding":"gzip:deflate:sdch:br",

        "x-user-agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36 FKUA/msite/0.0.1/msite/Mobile,Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36 FKUA/msite/0.0.1/msite/Mobile",

        "pragma":"no-cache",

        "issecureresource":"true"

}

class Store(MStore):

    OrderStatusMap = {

                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping', 'your order has been placed.'],

                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],

                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered', "item has been shipped.", "your item is out for delivery"],

                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.', 'your item has been cancelled', 'your item has been returned']

                      }

    def __init__(self,store_id):

        client = MongoClient('mongodb://localhost:27017/')

        self.db = client.dtr

        super(Store, self).__init__(store_id)

    def getName(self):

        return "flipkart"

    def _parseJSON(self,merchantOrder, transaction):

        merchantOrder = obj(merchantOrder)

        update = False

        if not hasattr(merchantOrder, 'subOrders'):

            merchantOrder.placedOn = transaction['orderDate'] 

            merchantOrder.paidAmount = transaction['amount']

            subOrders = []

            merchantOrder.subOrders = subOrders

            for shipment in transaction['groupedItems']:

                for item in shipment:

                    productUrl = "https://www.flipkart.com" + item['itemMetadata']['url']

                    merchantsubOrder = SubOrder(" ".join(filter(None, (item['title'], item['itemMetadata']['color']))), productUrl, merchantOrder.placedOn, item['amount'],MStore.ORDER_PLACED, item['quantity'])

                    merchantsubOrder.merchantSubOrderId = item['orderItemId']

                    merchantsubOrder.imgUrl  = item['itemMetadata']['images']['1000x1000']

                    productCode=productUrl.split("pid=")[1]

                    merchantsubOrder.productCode = productCode

                    merchantsubOrder.merchantSubOrderId = item['orderItemId']

                    merchantsubOrder.amount = item['itemSellingPrice']*merchantsubOrder.quantity

                    merchantsubOrder.unitPrice = item['itemSellingPrice']

                    merchantsubOrder.detailedStatus = item['desktopSubStatus']

                    subOrders.append(merchantsubOrder)

        else:

            update = True

            for shipment in transaction['groupedItems']:

                for item in shipment:

                    print "item----------", item

                    subOrder = self._isSubOrderActive1(merchantOrder,  item['orderItemId'])

                    if subOrder:

                        subOrder.detailedStatus = item['desktopSubStatus']

                        subOrder.status = self._getStatusFromDetailedStatus(subOrder.detailedStatus)

        self.populateDerivedFields(merchantOrder, update)

        self._updateToOrder(todict(merchantOrder))

    def _getActiveOrders(self, searchMap={}, collectionMap={}):

        collection = self.db.merchantOrder

        searchMap = dict(searchMap.items()+ {"closed": False, "storeId" : self.store_id}.items()) 

        #collectionMap =  dict(collectionMap.items() + {"orderSuccessUrl":1, "orderId":1,"subOrders":1, "placedOn":1, "orderTracking"}.items())

        stores = collection.find(searchMap)

        return [store for store in stores]    

    def trackOrdersForUser(self, userId, url, rawHtml):

        rawHtml = re.sub(r'[^\x00-\x7F]+',' ', rawHtml)

        headers = dict(TRACK_HEADERS)

        if url not in TRACK_URLS:

            for lis in re.findall('SN"?:\s?"(.*?)"', rawHtml):

                headers['sn'] = lis

                break

            for lis in re.findall('fkUA"?:\s?"(.*?)"', rawHtml):

                headers['User-Agent'] = lis.split(" FKUA")[0]

                headers['x-user-agent'] = lis

                break

            if self._getActiveOrders({"userId": userId,"createdOnInt":{"$gt":int(time.mktime(datetime.now().timetuple()))- 30*86400}}):

                return {"headers":headers, "urls":TRACK_URLS}

            else:

                return {"headers":headers, "urls":[]}

        else:

            jsonObject = json.loads(rawHtml)

            #Get only orders less than one month old

            activeOrders = self._getActiveOrders({"userId": userId, "createdOnInt":{"$gt":int(time.mktime(datetime.now().timetuple())) - 30*86400}})

            for order in activeOrders:

                for transaction in jsonObject['RESPONSE']['orderGranularDetails']:

                    if transaction['orderId'][:-2] == order["merchantOrderId"]:

                        self._parseJSON(order, transaction)

                        break

            return 'PARSED_SUCCESS'

    def requestDownload(self):

    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"

        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"

        br = getBrowserObject()

        br.set_debug_responses(True)

        br.open(AFFILIATE_URL)

        response = br.response()  # copy

        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]

        data = {

                    'j_username':'saholic1@gmail.com',

                    'j_password':'spice@2020'

            }

        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))

        till = datetime.strftime(date.today(),"%Y-%m-%d")

        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")

        #target = open("dowloadreportids", 'w')

        #target.truncate()

        for status in statuses:

            #try:

                data = {"till":till, "from":start, "filter":status}

                print json.dumps(data)

                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))

                print request

                response = utils.ungzipResponse(br.open(request))

                response = json.loads(response)

                print response

                if(response['status']=="OK"):

                    self.db.flipkartdownloadids.save(response)

                else:

                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")

                    #break 

            #except:

                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")

                #break 

    def scrapeStoreOrders(self,):

        pass

        orders = self._getActiveOrders()

        for order in orders:

            print "Order", self.store_name, order['orderId']

            try:

                closed = True

                url = ORDER_TRACK_URL + urlparse(order['orderSuccessUrl']).query    

                page = fetchResponseUsingProxy(url, headers)

                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

                sections = soup.findAll("div", {"class":"ui-app-card-body"})

                sections.pop(1)

                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})

                fkSubOrders = mainOrder.findAll("li")

                #remove unwanted list

                fkSubOrders.pop(-1)

                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()

                #fetching suborders details

                for subOrder in fkSubOrders:

                    updateMap = {}

                    ul = subOrder.find("ul")

                    if ul is None:

                        ul = subOrder.findAll("div", recursive=False)[0].div.div

                    orderItems = ul.findAll("div", recursive=False)

                    for orderItem in orderItems:

                        closedStatus = False

                        divs = orderItem.findAll('div', recursive=False)

                        orderTracking = divs[2]

                        merchantSubOrderId = divs[3].get('id')

                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)

                        if subOrder is None:

                            break

                        elif subOrder['closed']:

                            break

                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}

                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})

                        cashbackStatus = subOrder.get("cashBackStatus")

                        if orderTrackingDetDiv is not None:

                            orderTrackingDet = str(orderTrackingDetDiv.find("div", "tracking-remark").text)

                            updateMap["subOrders.$.detailedStatus"] = orderTrackingDet

                            tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                            tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})

                            if "approveDetails-ongoing" in str(tr) or "processingDetails-ongoing" in str(tr):

                                status=MStore.ORDER_PLACED

                            elif "shippingDetails-ongoing" in str(tr):

                                status = MStore.ORDER_SHIPPED

                            elif "delivery-complete" in str(tr):

                                status = MStore.ORDER_DELIVERED

                                if cashbackStatus != Store.CB_NA:

                                    cashbackStatus = Store.CB_APPROVED

                                closedStatus = True

                            if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):

                                status = MStore.ORDER_CANCELLED

                                closedStatus = True

                                if cashbackStatus == Store.CB_PENDING:

                                    cashbackStatus = Store.CB_CANCELLED

                        else:

                            updateMap["subOrders.$.detailedStatus"] = "Refunded"

                            status = MStore.ORDER_CANCELLED

                            closedStatus = True

                            if cashbackStatus == Store.CB_PENDING:

                                cashbackStatus = Store.CB_CANCELLED

                        print "Sub Order Status " + str(status)

                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus

                        updateMap["subOrders.$.status"] = status

                        updateMap["subOrders.$.closed"] = closedStatus

                        if closed:

                            closed = closedStatus

                        bulk.find(findMap).update({'$set' : updateMap})

                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})

                bulk.execute()            

            except:

                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})

                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())

                traceback.print_exc()

    def scrapeAffiliate(self, deltaDays=0):

        if deltaDays is None:

            deltaDays=0

        endDate=date.today() - timedelta(days=1)

        startDate = endDate - timedelta(days=deltaDays)

        endDate = datetime.strftime(endDate, "%Y-%m-%d")

        startDate = datetime.strftime(startDate, "%Y-%m-%d")

        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=0"

        for status in statuses:

            nextUrl = url%(startDate, endDate, status)

            while nextUrl:

                req = urllib2.Request(nextUrl)

                nextUrl=''

                req.add_header('Fk-Affiliate-Id', 'saholic1g')

                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')

                resp = urllib2.urlopen(req)

                resString = json.loads(resp.read())

                orderList = resString["orderList"]

                if orderList:

                    for order in orderList:

                        order['sales'] = int(order['sales']['amount'])

                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])

                        subTagId = order.get("affExtParam1")

                        userId = None

                        email = None

                        if subTagId:

                            click = session.query(Clicks).filter_by(tag = subTagId).first()

                            if click is not None:

                                userId= click.user_id 

                                user = session.query(Users.email).filter_by(id = userId).first()

                                if user is not None:

                                    email = user.email

                        flipkartOrder = FlipkartOrders()

                        flipkartOrder.subtagId = subTagId

                        flipkartOrder.user_id = userId

                        flipkartOrder.identifier = order.get("identifier")

                        flipkartOrder.email = email

                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%d-%m-%Y %H:%M:%S")

                        flipkartOrder.status = order.get("status")

                        flipkartOrder.title = order.get("title")

                        flipkartOrder.price = order.get("price")

                        flipkartOrder.quantity = order.get("quantity")

                        flipkartOrder.productCode = order.get("productId")

                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")

                        flipkartOrder.payOut = order['tentativeCommission']

                        flipkartOrder.payOutPercentage = order['commissionRate']

                        skuData = getSkuData(2, order.get("productId"))

                        if skuData is not None:

                            flipkartOrder.catalogId = skuData.get("skuBundleId")

                            flipkartOrder.brand = skuData.get("brand")

                            flipkartOrder.model = skuData.get("model_name")

                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))

                            flipkartOrder.title =skuData.get("source_product_name")

                    session.commit()

                    nextUrl = resString['next']  

    def _saveToAffiliate(self, offers):

        collection = self.db.flipkartOrderAffiliateInfo1

        count=0

        for row in offers:

            if count==0:

                count += 1

                continue

            offer = self.covertToObj(row)

            collection.save(offer)

    def covertToObj(self,offer):

        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer

        saleMap = {

               "affiliateorderitemid":affiliateorderitemid,

                "title":title,

                "productid":productid,

                "category":category,

                "quantity":quantity,

                "saleAmount":sales,

                "price":price,

                "commissionrate":commissionrate,

                "payOut":tentativecommission,

                "conversionStatus":status,

                "saleDate":orderdate,

                "saleschannel":saleschannel,

                "customertype":customertype,

                "affextparam1":affextparam1,

                "_id":affiliateorderitemid

        }

        return saleMap

    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):

        resp= {}

        try:

            url = ORDER_TRACK_URL + urlparse(orderSuccessUrl).query 

            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]

            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)

            merchantOrder.closed = False

            merchantOrder.merchantOrderId = merchantOrderId

            merchantOrder.orderTrackingUrl = url

            if self._saveToOrder(todict(merchantOrder)):

                resp['result'] = 'ORDER_CREATED'

            else:

                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'

            return resp 

        except:

            traceback.print_exc()

            resp['result'] = 'ORDER_NOT_CREATED'

            return resp 

    def _getStatusFromDetailedStatus(self, detailedStatus):

        for key, value in Store.OrderStatusMap.iteritems():

            if detailedStatus.lower() in value:

                return key

        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name

        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)           

def _saveToOrderFlipkart(self, order):

        collection = self.db.merchantOrder

        order = collection.insert(order)

def hex_md5(password):

    m = hashlib.md5()

    print(m.digest())

def main():

    store = getStore(2)

    store.trackOrdersForUser(30916, "https://www.flipkart.com/api/1/self-serve/orders?pageNum=1&link=myAccounts", readSSh("/home/amit/Downloads/htmldata"))

    #store.scrapeStoreOrders()

if __name__ == '__main__':

        main()