Subversion Repositories SmartDukaan

'''
Created on Jan 15, 2015

@author: amit
'''
from BeautifulSoup import BeautifulSoup
from datetime import datetime, date, timedelta
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
    Store as MStore, sourceMap, tprint
from dtr.storage.DataService import Clicks, Users, FlipkartOrders
from dtr.utils import utils
from dtr.utils.utils import fetchResponseUsingProxy, todict, getSkuData,\
    ORDER_PLACED
from elixir import *
from pprint import pprint
from pymongo.mongo_client import MongoClient
import StringIO
import csv
import hashlib
import importlib
import json
import mechanize
import pymongo
import re
import traceback
import urllib
import urllib2
from urlparse import urlparse
import zipfile
USERNAME='saholic1@gmail.com'
PASSWORD='spice@2020'
ORDER_TRACK_URL='https://m.flipkart.com/order_details?src=or&pr=1&type=physical&'
AFFILIATE_URL='https://affiliate.flipkart.com/'
AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'
AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"
AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"
AFF_STATUS_CANCELLED='cancelled'
AFF_STATUS_APPROVED='approved'
AFF_STATUS_DISAPPROVED='disapproved'
AFF_STATUS_PENDING='pending'
statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]
categoryMap = {3:"Mobiles", 5:"Tablets"}


class Store(MStore):
    OrderStatusMap = {
                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping'],
                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],
                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered'],
                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.']
                      
                      }
    def __init__(self,store_id):
        client = MongoClient('mongodb://localhost:27017/')
        self.db = client.dtr
        super(Store, self).__init__(store_id)

    def getName(self):
        return "flipkart"
    
    
    def requestDownload(self):
    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"
        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"
        br = getBrowserObject()
        br.set_debug_responses(True)
        br.open(AFFILIATE_URL)
        response = br.response()  # copy
        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]
        data = {
                    'j_username':'saholic1@gmail.com',
                    'j_password':'spice@2020'
            }
        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))
        till = datetime.strftime(date.today(),"%Y-%m-%d")
        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")
        #target = open("dowloadreportids", 'w')
        #target.truncate()
        for status in statuses:
            #try:
                data = {"till":till, "from":start, "filter":status}
                print json.dumps(data)
                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))
                print request
                response = utils.ungzipResponse(br.open(request))
                response = json.loads(response)
                print response
                if(response['status']=="OK"):
                    self.db.flipkartdownloadids.save(response)
                else:
                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")
                    #break 
            #except:
                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")
                #break 
                
    
    def scrapeStoreOrders(self,):
        orders = self._getActiveOrders()
        for order in orders:
            print "Order", self.store_name, order['orderId']
            try:
                closed = True
                url = ORDER_TRACK_URL + urlparse(order['orderSuccessUrl']).query    
                page = fetchResponseUsingProxy(url)
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
                
                sections = soup.findAll("div", {"class":"ui-app-card-body"})
                sections.pop(1)
        
                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
                fkSubOrders = mainOrder.findAll("li")
                
                #remove unwanted list
                fkSubOrders.pop(-1)
                
                
                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
                #fetching suborders details
                for subOrder in fkSubOrders:
                    updateMap = {}
                    ul = subOrder.find("ul")
                    if ul is None:
                        ul = subOrder.findAll("div", recursive=False)[0].div.div
                    orderItems = ul.findAll("div", recursive=False)
                    for orderItem in orderItems:
                        closedStatus = False
                        divs = orderItem.findAll('div', recursive=False)
                        orderTracking = divs[2]
                        merchantSubOrderId = divs[3].get('id')
                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
                        if subOrder is None:
                            break
                        elif subOrder['closed']:
                            break
                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})
                        cashbackStatus = subOrder.get("cashBackStatus")
                        if orderTrackingDetDiv is not None:
                            orderTrackingDet = str(orderTrackingDetDiv.find("div", "tracking-remark").text)
                            updateMap["subOrders.$.detailedStatus"] = orderTrackingDet
                            tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                            
                            tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
    
                            if "approveDetails-ongoing" in str(tr) or "processingDetails-ongoing" in str(tr):
                                status=MStore.ORDER_PLACED
                            elif "shippingDetails-ongoing" in str(tr):
                                status = MStore.ORDER_SHIPPED
                            elif "delivery-complete" in str(tr):
                                status = MStore.ORDER_DELIVERED
                                if cashbackStatus != Store.CB_NA:
                                    cashbackStatus = Store.CB_APPROVED
                                closedStatus = True
                            if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):
                                status = MStore.ORDER_CANCELLED
                                closedStatus = True
                                if cashbackStatus == Store.CB_PENDING:
                                    cashbackStatus = Store.CB_CANCELLED
                        else:
                            updateMap["subOrders.$.detailedStatus"] = "Refunded"
                            status = MStore.ORDER_CANCELLED
                            closedStatus = True
                            if cashbackStatus == Store.CB_PENDING:
                                cashbackStatus = Store.CB_CANCELLED
                        print "Sub Order Status " + str(status)
                        
                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus
                        updateMap["subOrders.$.status"] = status
                        updateMap["subOrders.$.closed"] = closedStatus
                        if closed:
                            closed = closedStatus
                        bulk.find(findMap).update({'$set' : updateMap})
                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})
                bulk.execute()            
            except:
                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())
                traceback.print_exc()
    
    def scrapeAffiliate(self, deltaDays=0):
        if deltaDays is None:
            deltaDays=0
        endDate=date.today() - timedelta(days=1)
        startDate = endDate - timedelta(days=deltaDays)
        
        endDate = datetime.strftime(endDate, "%Y-%m-%d")
        startDate = datetime.strftime(startDate, "%Y-%m-%d")
        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=0"
            
        for status in statuses:
            nextUrl = url%(startDate, endDate, status)
            while nextUrl:
                req = urllib2.Request(nextUrl)
                nextUrl=''
                req.add_header('Fk-Affiliate-Id', 'saholic1g')
                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')
                resp = urllib2.urlopen(req)
                resString = json.loads(resp.read())
                orderList = resString["orderList"]
                if orderList:
                    for order in orderList:
                        order['sales'] = int(order['sales']['amount'])
                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])
                        subTagId = order.get("affExtParam1")
                        userId = None
                        email = None
                        if subTagId:
                            click = session.query(Clicks).filter_by(tag = subTagId).first()
                            if click is not None:
                                userId= click.user_id 
                                user = session.query(Users.email).filter_by(id = userId).first()
                                if user is not None:
                                    email = user.email
                        
                        flipkartOrder = FlipkartOrders()
                        flipkartOrder.subtagId = subTagId
                        flipkartOrder.user_id = userId
                        flipkartOrder.identifier = order.get("identifier")
                        flipkartOrder.email = email
                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%d-%m-%Y %H:%M:%S")
                        flipkartOrder.status = order.get("status")
                        flipkartOrder.title = order.get("title")
                        flipkartOrder.price = order.get("price")
                        flipkartOrder.quantity = order.get("quantity")
                        flipkartOrder.productCode = order.get("productId")
                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")
                        flipkartOrder.payOut = order['tentativeCommission']
                        flipkartOrder.payOutPercentage = order['commissionRate']
                        skuData = getSkuData(2, order.get("productId"))
                        if skuData is not None:
                            flipkartOrder.catalogId = skuData.get("skuBundleId")
                            flipkartOrder.brand = skuData.get("brand")
                            flipkartOrder.model = skuData.get("model_name")
                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))
                            flipkartOrder.title =skuData.get("source_product_name")
                                    
                    session.commit()
                    nextUrl = resString['next']  
    
    def _saveToAffiliate(self, offers):
        collection = self.db.flipkartOrderAffiliateInfo1
        count=0
        for row in offers:
            if count==0:
                count += 1
                continue
            offer = self.covertToObj(row)
            collection.save(offer)
    
    def covertToObj(self,offer):
        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer
        saleMap = {
               "affiliateorderitemid":affiliateorderitemid,
                "title":title,
                "productid":productid,
                "category":category,
                "quantity":quantity,
                "saleAmount":sales,
                "price":price,
                "commissionrate":commissionrate,
                "payOut":tentativecommission,
                "conversionStatus":status,
                "saleDate":orderdate,
                "saleschannel":saleschannel,
                "customertype":customertype,
                "affextparam1":affextparam1,
                "_id":affiliateorderitemid
        }
        return saleMap
            
            
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        resp= {}
        try:
            br = getBrowserObject()
            url = ORDER_TRACK_URL + urlparse(orderSuccessUrl).query 
           
            response = br.open(url)
            page = ungzipResponse(response)
            
            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]
            print merchantOrderId
            
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            
            sections = soup.findAll("div", {"class":"ui-app-card-body"})
            sections.pop(1)
    
            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
            merchantOrder.closed = True
            merchantOrder.orderTrackingUrl = url
            for data in sections:
                name = data.findAll("span")
                i=0
                while i< len(name):
                    if "key" in str(name[i]):
                        if "Grand Total" in str(name[i]):
                            #Total Amount
                            merchantOrder.paidAmount = int(re.findall(r'\d+', name[i+1].text)[0])
                        elif "Order Date" in str(name[i]):
                            merchantOrder.placedOn = name[i+1].text 
                    i=i+1    
            
            merchantOrder.merchantOrderId = merchantOrderId
    
            mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
            fkSubOrders = mainOrder.findAll("li")
            #remove unwanted list
            fkSubOrders.pop(-1)
            subOrders = []
            merchantOrder.subOrders = subOrders
    
            for subOrder in fkSubOrders:
                orderItems = subOrder.findAll("div", "product-list-item", recursive=True)
                for orderItem in orderItems:
                    if not orderItem.has_key("id"):
                        continue
                    divs = orderItem.findAll('div', recursive=False)
                    content = divs[0]
                    orderTracking = divs[2]
                    merchantSubOrderId = divs[3].get('id')
                    imgUrl = content.find('img')['src']
                    lineDet = content.find("div",{"class":"product-info"})
                    productTitle = lineDet.find("a",{"class":"product-title"}).text
                    productUrl =  str(lineDet.find("a")['href'])
    
                    start='pid='
                    s=str(lineDet.find("a")['href'])
                    productCode = re.findall(re.escape(start)+"(.*)",s)[0].strip()
                    mname = lineDet.findAll("span")
                    k=0
                    while k<len(mname):
                        if "note" in str(mname[k]):
                            if "Color:" in str(mname[k]):
                                productTitle = productTitle + " " + mname[k+1].text
                                
                            elif "Qty:" in str(mname[k]):
                                quantity = int(mname[k+1].text)
                            elif "Subtotal:" in str(mname[k]):
                                amountPaid = int(re.findall(r'\d+', mname[k+1].text)[0])
                            elif "Delivery:" in str(mname[k]):
                                #Delivery Date for sub order
                                print "Delivery Date " +mname[k+1].text
                        k=k+1
                    merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)
                    merchantsubOrder.imgUrl  = imgUrl
                    merchantsubOrder.productCode = productCode
                    merchantsubOrder.merchantSubOrderId = merchantSubOrderId
                    print "productCode", productCode
                    print "amountPaid", amountPaid
                    cashbackAmount, cashbackPercent = self.getCashbackAmount(productCode, amountPaid)
                    cashbackStatus = Store.CB_PENDING
                    if cashbackAmount <= 0:
                        cashbackStatus = Store.CB_NA
                    merchantsubOrder.cashBackAmount = cashbackAmount
                    merchantsubOrder.cashBackStatus = cashbackStatus      
                    merchantsubOrder.cashBackPercentage = cashbackPercent
                    activePart = orderItem.find("div", "c-tabs-content m-top active")
                    if activePart is not None:
                        merchantsubOrder.detailedStatus = str(activePart.find("div", "tracking-remark").text)   
                    
                    #To track shipping details         
                    status=ORDER_PLACED
                    tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                    if "approveDetails-ongoing" in str(tr) or "processingDetails-ongoing" in str(tr):
                        pass
                    elif "shippingDetails-ongoing" in str(tr):
                        status = MStore.ORDER_SHIPPED
                    elif "delivery-complete" in str(tr):
                        status = MStore.ORDER_DELIVERED
                        if cashbackStatus!=MStore.CB_NA:
                            merchantsubOrder.cashBackStatus = MStore.CB_APPROVED
                        merchantsubOrder.closed = True
                    if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):
                        status = MStore.ORDER_CANCELLED
                        if merchantsubOrder.cashBackStatus != MStore.CB_NA:    
                            merchantsubOrder.cashBackStatus = MStore.CB_CANCELLED
                        merchantsubOrder.closed = True
                    print "Sub Order Status " + str(status)
                    merchantsubOrder.status=status
                    if merchantOrder.closed:
                        merchantOrder.closed = merchantsubOrder.closed 
                    subOrders.append(merchantsubOrder)

            if self._saveToOrder(todict(merchantOrder)):
                resp['result'] = 'ORDER_CREATED'
            else:
                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                
            return resp 
        except:
            traceback.print_exc()
            resp['result'] = 'ORDER_NOT_CREATED'
            return resp 
            
    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, value in Store.OrderStatusMap.iteritems():
            if detailedStatus.lower() in value:
                return key
        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)           
                
def _saveToOrderFlipkart(self, order):
        collection = self.db.merchantOrder
        order = collection.insert(order)


    
def hex_md5(password):
    m = hashlib.md5()
    print(m.digest())

def main():
    store = getStore(2)
    #store.parseOrderRawHtml(1, '123', 14, '', '    https://www.flipkart.com/rv/orderConfirmation/orderresponse?reference_id=OD4072085355271530&token=c8a726cb1495232e00338fb0eecb4f40')
    store.scrapeStoreOrders()

if __name__ == '__main__':
        main()