WebSVN – SmartDukaan – /trunk/PyDTR/src/dtr/sources/flipkart.py

'''
Created on Jan 15, 2015

@author: amit
'''
from BeautifulSoup import BeautifulSoup
from datetime import datetime, date, timedelta
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo, obj
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
    Store as MStore, sourceMap, tprint
from dtr.storage.DataService import Clicks, Users, FlipkartOrders, OrdersRaw
from dtr.utils import utils
from dtr.utils.utils import fetchResponseUsingProxy, todict, getSkuData, \
    ORDER_PLACED, readSSh
from elixir import *
from pprint import pprint
from pymongo.mongo_client import MongoClient
from urlparse import urlparse
import StringIO
import csv
import hashlib
import importlib
import json
import mechanize
import pymongo
import re
import time
import traceback
import urllib
import urllib2
import zipfile
USERNAME='saholic1@gmail.com'
PASSWORD='spice@2020'
ORDER_TRACK_URL='https://www.flipkart.com/order_details?src=or&pr=1&type=physical&'
AFFILIATE_URL='https://affiliate.flipkart.com/'
AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'
AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"
AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"
AFF_STATUS_CANCELLED='cancelled'
AFF_STATUS_APPROVED='approved'
AFF_STATUS_DISAPPROVED='disapproved'
AFF_STATUS_PENDING='pending'
statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]
categoryMap = {3:"Mobiles", 5:"Tablets"}
mobileAgent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4'

headers = { 
            'User-Agent':'Mozilla/5.0 (iPhone; CPU iPhone OS 8_0 like Mac OS X) AppleWebKit/600.1.3 (KHTML, like Gecko) Version/8.0 Mobile/12A4345d Safari/600.1.4',
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',      
            'Accept-Language' : 'en-US,en;q=0.8',                     
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Connection':'keep-alive',
            'Accept-Encoding' : 'gzip,deflate,sdch'
        }

TRACK_URLS = ['https://www.flipkart.com/api/1/self-serve/orders?pageNum=1&link=myAccounts', 'https://www.flipkart.com/api/1/self-serve/orders?pageNum=2&link=myAccounts', 'https://www.flipkart.com/api/1/self-serve/orders?pageNum=3&link=myAccounts']
TRACK_HEADERS = {
        "sn":"2.VIBAC8BD9D21024F649B0482FAC385814A.SIE6C5F25D78974A5EB850D89167D52009.VSE7C76F85C2EE463786F89407CA46CA31.1492756338",
        "Connection":"keep-alive",
        "Referer":"https://www.flipkart.com/rv/orders?link=myAccounts",
        "cache-control":"no-cache",
        "Accept":"/",
        "Accept-Language":"en-US,en;q=0.8",
        "content-type":"application/json",
        "User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36",
        "flipkart_secure":"true",
        "sc":"",
        "Accept-Encoding":"gzip:deflate:sdch:br",
        "x-user-agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36 FKUA/msite/0.0.1/msite/Mobile,Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML:like Gecko) Chrome/48.0.2564.23 Mobile Safari/537.36 FKUA/msite/0.0.1/msite/Mobile",
        "pragma":"no-cache",
        "issecureresource":"true"
}


class Store(MStore):
    OrderStatusMap = {
                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping', 'your order has been placed.'],
                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],
                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered', "item has been shipped.", "your item is out for delivery"],
                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.', 'your item has been cancelled', 'your item has been returned']
                      
                      }
    def __init__(self,store_id):
        client = MongoClient('mongodb://localhost:27017/')
        self.db = client.dtr
        super(Store, self).__init__(store_id)

    def getName(self):
        return "flipkart"
    
    def _parseJSON(self,merchantOrder, transaction):
        merchantOrder = obj(merchantOrder)
        update = False
        if not hasattr(merchantOrder, 'subOrders'):
            merchantOrder.placedOn = transaction['orderDate'] 
            merchantOrder.paidAmount = transaction['amount']
            subOrders = []
            merchantOrder.subOrders = subOrders
            for shipment in transaction['groupedItems']:
                for item in shipment:
                    productUrl = "https://www.flipkart.com" + item['itemMetadata']['url']
                    merchantsubOrder = SubOrder(" ".join(filter(None, (item['title'], item['itemMetadata']['color']))), productUrl, merchantOrder.placedOn, item['amount'],MStore.ORDER_PLACED, item['quantity'])
                    merchantsubOrder.merchantSubOrderId = item['orderItemId']
                    merchantsubOrder.imgUrl  = item['itemMetadata']['images']['1000x1000']
                    productCode=productUrl.split("pid=")[1]
                    merchantsubOrder.productCode = productCode
                    merchantsubOrder.merchantSubOrderId = item['orderItemId']
                    merchantsubOrder.amount = item['itemSellingPrice']*merchantsubOrder.quantity
                    merchantsubOrder.unitPrice = item['itemSellingPrice']
                    merchantsubOrder.detailedStatus = item['desktopSubStatus']
                    subOrders.append(merchantsubOrder)
        else:
            update = True
            for shipment in transaction['groupedItems']:
                for item in shipment:
                    print "item----------", item
                    subOrder = self._isSubOrderActive1(merchantOrder,  item['orderItemId'])
                    if subOrder:
                        subOrder.detailedStatus = item['desktopSubStatus']
                        subOrder.status = self._getStatusFromDetailedStatus(subOrder.detailedStatus)
        
        self.populateDerivedFields(merchantOrder, update)
        self._updateToOrder(todict(merchantOrder))
    
    def _getActiveOrders(self, searchMap={}, collectionMap={}):
        collection = self.db.merchantOrder
        searchMap = dict(searchMap.items()+ {"closed": False, "storeId" : self.store_id}.items()) 
        #collectionMap =  dict(collectionMap.items() + {"orderSuccessUrl":1, "orderId":1,"subOrders":1, "placedOn":1, "orderTracking"}.items())
        stores = collection.find(searchMap)
        return [store for store in stores]    
    
    
    def trackOrdersForUser(self, userId, url, rawHtml):
        rawHtml = re.sub(r'[^\x00-\x7F]+',' ', rawHtml)
        headers = dict(TRACK_HEADERS)
        if url not in TRACK_URLS:
            for lis in re.findall('SN"?:\s?"(.*?)"', rawHtml):
                headers['sn'] = lis
                break
            for lis in re.findall('fkUA"?:\s?"(.*?)"', rawHtml):
                headers['User-Agent'] = lis.split(" FKUA")[0]
                headers['x-user-agent'] = lis
                break

            if self._getActiveOrders({"userId": userId,"createdOnInt":{"$gt":int(time.mktime(datetime.now().timetuple()))- 30*86400}}):
                return {"headers":headers, "urls":TRACK_URLS}
            else:
                return {"headers":headers, "urls":[]}
        else:
            jsonObject = json.loads(rawHtml)
            #Get only orders less than one month old
            activeOrders = self._getActiveOrders({"userId": userId, "createdOnInt":{"$gt":int(time.mktime(datetime.now().timetuple())) - 30*86400}})
            for order in activeOrders:
                for transaction in jsonObject['RESPONSE']['orderGranularDetails']:
                    if transaction['orderId'][:-2] == order["merchantOrderId"]:
                        self._parseJSON(order, transaction)
                        break
                
                
                
            return 'PARSED_SUCCESS'

    
    def requestDownload(self):
    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"
        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"
        br = getBrowserObject()
        br.set_debug_responses(True)
        br.open(AFFILIATE_URL)
        response = br.response()  # copy
        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]
        data = {
                    'j_username':'saholic1@gmail.com',
                    'j_password':'spice@2020'
            }
        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))
        till = datetime.strftime(date.today(),"%Y-%m-%d")
        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")
        #target = open("dowloadreportids", 'w')
        #target.truncate()
        for status in statuses:
            #try:
                data = {"till":till, "from":start, "filter":status}
                print json.dumps(data)
                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))
                print request
                response = utils.ungzipResponse(br.open(request))
                response = json.loads(response)
                print response
                if(response['status']=="OK"):
                    self.db.flipkartdownloadids.save(response)
                else:
                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")
                    #break 
            #except:
                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")
                #break 
                
    
    def scrapeStoreOrders(self,):
        pass
        orders = self._getActiveOrders()
        for order in orders:
            print "Order", self.store_name, order['orderId']
            try:
                closed = True
                url = ORDER_TRACK_URL + urlparse(order['orderSuccessUrl']).query    
                page = fetchResponseUsingProxy(url, headers)
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
                
                sections = soup.findAll("div", {"class":"ui-app-card-body"})
                sections.pop(1)
        
                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
                fkSubOrders = mainOrder.findAll("li")
                
                #remove unwanted list
                fkSubOrders.pop(-1)
                
                
                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
                #fetching suborders details
                for subOrder in fkSubOrders:
                    updateMap = {}
                    ul = subOrder.find("ul")
                    if ul is None:
                        ul = subOrder.findAll("div", recursive=False)[0].div.div
                    orderItems = ul.findAll("div", recursive=False)
                    for orderItem in orderItems:
                        closedStatus = False
                        divs = orderItem.findAll('div', recursive=False)
                        orderTracking = divs[2]
                        merchantSubOrderId = divs[3].get('id')
                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
                        if subOrder is None:
                            break
                        elif subOrder['closed']:
                            break
                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})
                        cashbackStatus = subOrder.get("cashBackStatus")
                        if orderTrackingDetDiv is not None:
                            orderTrackingDet = str(orderTrackingDetDiv.find("div", "tracking-remark").text)
                            updateMap["subOrders.$.detailedStatus"] = orderTrackingDet
                            tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                            
                            tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
    
                            if "approveDetails-ongoing" in str(tr) or "processingDetails-ongoing" in str(tr):
                                status=MStore.ORDER_PLACED
                            elif "shippingDetails-ongoing" in str(tr):
                                status = MStore.ORDER_SHIPPED
                            elif "delivery-complete" in str(tr):
                                status = MStore.ORDER_DELIVERED
                                if cashbackStatus != Store.CB_NA:
                                    cashbackStatus = Store.CB_APPROVED
                                closedStatus = True
                            if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):
                                status = MStore.ORDER_CANCELLED
                                closedStatus = True
                                if cashbackStatus == Store.CB_PENDING:
                                    cashbackStatus = Store.CB_CANCELLED
                        else:
                            updateMap["subOrders.$.detailedStatus"] = "Refunded"
                            status = MStore.ORDER_CANCELLED
                            closedStatus = True
                            if cashbackStatus == Store.CB_PENDING:
                                cashbackStatus = Store.CB_CANCELLED
                        print "Sub Order Status " + str(status)
                        
                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus
                        updateMap["subOrders.$.status"] = status
                        updateMap["subOrders.$.closed"] = closedStatus
                        if closed:
                            closed = closedStatus
                        bulk.find(findMap).update({'$set' : updateMap})
                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})
                bulk.execute()            
            except:
                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())
                traceback.print_exc()
    
    def scrapeAffiliate(self, deltaDays=0):
        if deltaDays is None:
            deltaDays=0
        endDate=date.today() - timedelta(days=1)
        startDate = endDate - timedelta(days=deltaDays)
        
        endDate = datetime.strftime(endDate, "%Y-%m-%d")
        startDate = datetime.strftime(startDate, "%Y-%m-%d")
        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=0"
            
        for status in statuses:
            nextUrl = url%(startDate, endDate, status)
            while nextUrl:
                req = urllib2.Request(nextUrl)
                nextUrl=''
                req.add_header('Fk-Affiliate-Id', 'saholic1g')
                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')
                resp = urllib2.urlopen(req)
                resString = json.loads(resp.read())
                orderList = resString["orderList"]
                if orderList:
                    for order in orderList:
                        order['sales'] = int(order['sales']['amount'])
                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])
                        subTagId = order.get("affExtParam1")
                        userId = None
                        email = None
                        if subTagId:
                            click = session.query(Clicks).filter_by(tag = subTagId).first()
                            if click is not None:
                                userId= click.user_id 
                                user = session.query(Users.email).filter_by(id = userId).first()
                                if user is not None:
                                    email = user.email
                        
                        flipkartOrder = FlipkartOrders()
                        flipkartOrder.subtagId = subTagId
                        flipkartOrder.user_id = userId
                        flipkartOrder.identifier = order.get("identifier")
                        flipkartOrder.email = email
                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%d-%m-%Y %H:%M:%S")
                        flipkartOrder.status = order.get("status")
                        flipkartOrder.title = order.get("title")
                        flipkartOrder.price = order.get("price")
                        flipkartOrder.quantity = order.get("quantity")
                        flipkartOrder.productCode = order.get("productId")
                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")
                        flipkartOrder.payOut = order['tentativeCommission']
                        flipkartOrder.payOutPercentage = order['commissionRate']
                        skuData = getSkuData(2, order.get("productId"))
                        if skuData is not None:
                            flipkartOrder.catalogId = skuData.get("skuBundleId")
                            flipkartOrder.brand = skuData.get("brand")
                            flipkartOrder.model = skuData.get("model_name")
                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))
                            flipkartOrder.title =skuData.get("source_product_name")
                                    
                    session.commit()
                    nextUrl = resString['next']  
    
    def _saveToAffiliate(self, offers):
        collection = self.db.flipkartOrderAffiliateInfo1
        count=0
        for row in offers:
            if count==0:
                count += 1
                continue
            offer = self.covertToObj(row)
            collection.save(offer)
    
    def covertToObj(self,offer):
        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer
        saleMap = {
               "affiliateorderitemid":affiliateorderitemid,
                "title":title,
                "productid":productid,
                "category":category,
                "quantity":quantity,
                "saleAmount":sales,
                "price":price,
                "commissionrate":commissionrate,
                "payOut":tentativecommission,
                "conversionStatus":status,
                "saleDate":orderdate,
                "saleschannel":saleschannel,
                "customertype":customertype,
                "affextparam1":affextparam1,
                "_id":affiliateorderitemid
        }
        return saleMap
            
            
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        resp= {}
        try:
            url = ORDER_TRACK_URL + urlparse(orderSuccessUrl).query 
            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]
        
            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
            merchantOrder.closed = False
            merchantOrder.merchantOrderId = merchantOrderId
            merchantOrder.orderTrackingUrl = url
            if self._saveToOrder(todict(merchantOrder)):
                resp['result'] = 'ORDER_CREATED'
            else:
                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                
            return resp 
        except:
            traceback.print_exc()
            resp['result'] = 'ORDER_NOT_CREATED'
            return resp 
        
 
            
    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, value in Store.OrderStatusMap.iteritems():
            if detailedStatus.lower() in value:
                return key
        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)           
                
def _saveToOrderFlipkart(self, order):
        collection = self.db.merchantOrder
        order = collection.insert(order)


    
def hex_md5(password):
    m = hashlib.md5()
    print(m.digest())

def main():
    store = getStore(2)
    store.trackOrdersForUser(30916, "https://www.flipkart.com/api/1/self-serve/orders?pageNum=1&link=myAccounts", readSSh("/home/amit/Downloads/htmldata"))
    #store.scrapeStoreOrders()

if __name__ == '__main__':
        main()
Subversion Repositories SmartDukaan

(root)/trunk/PyDTR/src/dtr/sources/flipkart.py – Rev 21816
