Subversion Repositories SmartDukaan

'''
Created on Jan 15, 2015

@author: amit
'''
from BeautifulSoup import BeautifulSoup
from datetime import datetime, date, timedelta
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
    Store as MStore, sourceMap, tprint
from dtr.reports.affiliatereco import getSkuData
from dtr.storage.DataService import Clicks, Users, FlipkartOrders
from dtr.utils import utils
from dtr.utils.utils import fetchResponseUsingProxy
from elixir import *
from pprint import pprint
from pymongo.mongo_client import MongoClient
import StringIO
import csv
import hashlib
import importlib
import json
import mechanize
import pymongo
import re
import traceback
import urllib
import zipfile
import urllib2
USERNAME='saholic1@gmail.com'
PASSWORD='spice@2020'
ORDER_TRACK_URL='https://m.flipkart.com/order_details'
AFFILIATE_URL='https://affiliate.flipkart.com/'
AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'
AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"
AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"
AFF_STATUS_CANCELLED='cancelled'
AFF_STATUS_APPROVED='approved'
AFF_STATUS_DISAPPROVED='disapproved'
AFF_STATUS_PENDING='pending'
statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]
categoryMap = {3:"Mobiles", 5:"Tablets"}


class Store(MStore):
    OrderStatusMap = {
                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping'],
                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],
                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered'],
                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.']
                      
                      }
    def __init__(self,store_id):
        client = MongoClient('mongodb://localhost:27017/')
        self.db = client.dtr
        super(Store, self).__init__(store_id)

    def getName(self):
        return "flipkart"
    
    
    def requestDownload(self):
    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"
        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"
        br = getBrowserObject()
        br.set_debug_responses(True)
        br.open(AFFILIATE_URL)
        response = br.response()  # copy
        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]
        data = {
                    'j_username':'saholic1@gmail.com',
                    'j_password':'spice@2020'
            }
        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))
        till = datetime.strftime(date.today(),"%Y-%m-%d")
        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")
        #target = open("dowloadreportids", 'w')
        #target.truncate()
        for status in statuses:
            #try:
                data = {"till":till, "from":start, "filter":status}
                print json.dumps(data)
                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))
                print request
                response = utils.ungzipResponse(br.open(request))
                response = json.loads(response)
                print response
                if(response['status']=="OK"):
                    self.db.flipkartdownloadids.save(response)
                else:
                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")
                    #break 
            #except:
                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")
                #break 
                
    
    def scrapeStoreOrders(self,):
        orders = self._getActiveOrders()
        for order in orders:
            print "Order", self.store_name, order['orderId']
            try:
                closed = True
                url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
                page = fetchResponseUsingProxy(url)
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
                
                sections = soup.findAll("div", {"class":"ui-app-card-body"})
                sections.pop(1)
        
                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
                fkSubOrders = mainOrder.findAll("li")
                
                #remove unwanted list
                fkSubOrders.pop(-1)
                
                
                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
                #fetching suborders details
                for subOrder in fkSubOrders:
                    updateMap = {}
                    y = subOrder.find("header").findAll("span")
                    for suborderId in y:
                        if "value emp" in str(suborderId):
                            merchantSubOrderId = suborderId.text
                            break
                    ul = subOrder.find("ul")
                    if ul is None:
                        ul = subOrder.findAll("div", recursive=False)[0].div.div
                    orderItems = ul.findAll("div", recursive=False)
                    i=0
                    for orderItem in orderItems:
                        closedStatus = False
                        merchantSubOrderId = merchantSubOrderId + str(i)
                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
                        if subOrder is None:
                            break
                        elif subOrder['closed']:
                            break
                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
                        divs = orderItem.findAll('div', recursive=False)
                        orderTracking = divs[2]
                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})
                        orderTrackingDet = orderTrackingDetDiv.find('div',{'class':re.compile('tracking-remark')}).text
                        updateMap["subOrders.$.detailedStatus"] = orderTrackingDet
                        status = self._getStatusFromDetailedStatus(orderTrackingDet)
                        tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                        
                        cashbackStatus = subOrder.get("cashBackStatus") 
                        if "approveDetails-complete" in str(tr):
                            if "processingDetails-complete" in str(tr):
                                if "shippingDetails-complete" in str(tr):
                                    if "delivery-complete" in str(tr):
                                        status = MStore.ORDER_DELIVERED
                                        if cashbackStatus == Store.CB_PENDING:
                                            cashbackStatus = Store.CB_APPROVED
                                        closedStatus = True
                                    else:     
                                        status = MStore.ORDER_SHIPPED
                                        courierTrackAnchor = orderTrackingDetDiv.find('a')
                                        trackingUrl = courierTrackAnchor("href")
                                        trackingText = courierTrackAnchor.text
                                        courierArr = trackingText.split(' : ')
                                        updateMap["subOrders.$.trackingUrl"] = trackingUrl
                                        updateMap["subOrders.$.trackingNumber"] = courierArr[1]
                                        updateMap["subOrders.$.courierName"] = courierArr[0]
                                else:
                                    status = MStore.ORDER_CANCELLED
                                    closedStatus = True
                                    if cashbackStatus == Store.CB_PENDING:
                                        cashbackStatus = Store.CB_CANCELLED
                            else: 
                                status = MStore.ORDER_PLACED
                        elif str(tr) in ["approveDetails-ongoing"]:
                            status=MStore.ORDER_PLACED             
                        if "dead" in str(tr) or status==MStore.ORDER_CANCELLED:
                            status = MStore.ORDER_CANCELLED
                            closedStatus = True
                            if cashbackStatus == Store.CB_PENDING:
                                cashbackStatus = Store.CB_CANCELLED
                        
                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus
                        updateMap["subOrders.$.status"] = status
                        updateMap["subOrders.$.closed"] = closedStatus
                        if closed:
                            closed = closedStatus
                        bulk.find(findMap).update({'$set' : updateMap})
                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})
                bulk.execute()            
            except:
                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())
                traceback.print_exc()
    
    def scrapeAffiliate(self, deltaDays=0):
        if deltaDays is None:
            deltaDays=0
        endDate=date.today() - timedelta(days=1)
        startDate = endDate - timedelta(days=deltaDays)
        
        endDate = datetime.strftime(endDate, "%Y-%m-%d")
        startDate = datetime.strftime(startDate, "%Y-%m-%d")
        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=0"
            
        for status in statuses:
            nextUrl = url%(startDate, endDate, status)
            while nextUrl:
                req = urllib2.Request(nextUrl)
                nextUrl=''
                req.add_header('Fk-Affiliate-Id', 'saholic1g')
                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')
                resp = urllib2.urlopen(req)
                resString = json.loads(resp.read())
                orderList = resString["orderList"]
                if orderList:
                    for order in orderList:
                        order['sales'] = int(order['sales']['amount'])
                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])
                        subTagId = order.get("affExtParam1")
                        userId = None
                        email = None
                        if subTagId:
                            click = session.query(Clicks).filter_by(tag = subTagId).first()
                            if click is not None:
                                userId= click.user_id 
                                user = session.query(Users.email).filter_by(id = userId).first()
                                if user is not None:
                                    email = user.email
                        
                        flipkartOrder = FlipkartOrders()
                        flipkartOrder.subtagId = subTagId
                        flipkartOrder.user_id = userId
                        flipkartOrder.identifier = order.get("identifier")
                        flipkartOrder.email = email
                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%d-%m-%Y")
                        flipkartOrder.status = order.get("status")
                        flipkartOrder.title = order.get("title")
                        flipkartOrder.price = order.get("price")
                        flipkartOrder.quantity = order.get("quantity")
                        flipkartOrder.productCode = order.get("productId")
                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")
                        flipkartOrder.payOut = order['tentativeCommission']
                        flipkartOrder.payOutPercentage = order['commissionRate']
                        skuData = getSkuData(2, order.get("productId"))
                        if skuData is not None:
                            flipkartOrder.catalogId = skuData.get("skuBundleId")
                            flipkartOrder.brand = skuData.get("brand")
                            flipkartOrder.model = skuData.get("model_name")
                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))
                            flipkartOrder.title =skuData.get("source_product_name")
                                    
                    session.commit()
                    nextUrl = resString['next']  
    
    def _saveToAffiliate(self, offers):
        collection = self.db.flipkartOrderAffiliateInfo1
        count=0
        for row in offers:
            if count==0:
                count += 1
                continue
            offer = self.covertToObj(row)
            collection.save(offer)
    
    def covertToObj(self,offer):
        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer
        saleMap = {
               "affiliateorderitemid":affiliateorderitemid,
                "title":title,
                "productid":productid,
                "category":category,
                "quantity":quantity,
                "saleAmount":sales,
                "price":price,
                "commissionrate":commissionrate,
                "payOut":tentativecommission,
                "conversionStatus":status,
                "saleDate":orderdate,
                "saleschannel":saleschannel,
                "customertype":customertype,
                "affextparam1":affextparam1,
                "_id":affiliateorderitemid
        }
        return saleMap
            
            
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        resp= {}
        try:
            br = getBrowserObject()
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
           
            response = br.open(url)
            page = ungzipResponse(response)
            
            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]
            print merchantOrderId
            
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            
            sections = soup.findAll("div", {"class":"ui-app-card-body"})
            sections.pop(1)
    
            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
            merchantOrder.orderTrackingUrl = url
            for data in sections:
                name = data.findAll("span")
                i=0
                while i< len(name):
                    if "key" in str(name[i]):
                        if "Grand Total" in str(name[i]):
                            #Total Amount
                            merchantOrder.paidAmount = int(re.findall(r'\d+', name[i+1].text)[0])
                        elif "Order Date" in str(name[i]):
                            merchantOrder.placedOn = name[i+1].text 
                    i=i+1    
            
            merchantOrder.merchantOrderId = merchantOrderId
    
            mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
            fkSubOrders = mainOrder.findAll("li")
            #remove unwanted list
            fkSubOrders.pop(-1)
            subOrders = []
            merchantOrder.subOrders = subOrders
    
            #fetching suborders details
            for subOrder in fkSubOrders:
                y = subOrder.find("header").findAll("span")
                for suborderId in y:
                    if "value emp" in str(suborderId):
                        merchantSubOrderId = suborderId.text
                        break
                orderItems = subOrder.find("ul").findAll("div", recursive=False)
                i=0
                for orderItem in orderItems:
                    merchantOrder.placedOn
                    merchantSubOrderId = merchantSubOrderId + str(i)
                    divs = orderItem.findAll('div', recursive=False)
                    content = divs[0]
                    orderTracking = divs[2]
                    orderTrackingDet = divs[3]
                    
                    
                    imgUrl = content.find('img')['src']
                    lineDet = content.find("div",{"class":"product-info"})
                    productTitle = lineDet.find("a",{"class":"product-title"}).text
                    productUrl =  str(lineDet.find("a")['href'])
    
                    start='pid='
                    s=str(lineDet.find("a")['href'])
                    productCode = re.findall(re.escape(start)+"(.*)",s)[0].strip()
                    mname = lineDet.findAll("span")
                    k=0
                    while k<len(mname):
                        if "note" in str(mname[k]):
                            if "Color:" in str(mname[k]):
                                productTitle = productTitle + " " + mname[k+1].text
                                
                            elif "Qty:" in str(mname[k]):
                                quantity = int(mname[k+1].text)
                            elif "Subtotal:" in str(mname[k]):
                                amountPaid = int(re.findall(r'\d+', mname[k+1].text)[0])
                            elif "Delivery:" in str(mname[k]):
                                #Delivery Date for sub order
                                print "Delivery Date " +mname[k+1].text
                        k=k+1
                    merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)
                    merchantsubOrder.imgUrl  = imgUrl
                    merchantsubOrder.productCode = productCode
                    merchantsubOrder.merchantSubOrderId = merchantSubOrderId
                    print "productCode", productCode
                    print "amountPaid", amountPaid
                    cashbackAmount, cashbackPercent = self.getCashbackAmount(productCode, amountPaid)
                    cashbackStatus = Store.CB_PENDING
                    if cashbackAmount <= 0:
                        cashbackStatus = Store.CB_NA
                    merchantsubOrder.cashBackAmount = cashbackAmount
                    merchantsubOrder.cashBackStatus = cashbackStatus      
                    merchantsubOrder.cashBackPercentage = cashbackPercent      
                    
                    subOrders.append(merchantsubOrder)
                    #To track shipping details         
                    status=-1    
                    tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                    if "approveDetails-complete" in str(tr):
                        if "processingDetails-complete" in str(tr):
                            if "shippingDetails-complete" in str(tr):
                                if "delivery-complete" in str(tr):
                                    status = MStore.ORDER_DELIVERED
                                else:     
                                    status = MStore.ORDER_SHIPPED
                                    orderTracking.find('div', {'class':'tracking-remark active'})
                            else:
                                status = MStore.ORDER_PLACED
                        else: 
                            status = MStore.ORDER_PLACED
                    elif str(tr) in ["approveDetails-ongoing"]:
                        status=MStore.ORDER_PLACED             
                    if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):
                        status = MStore.ORDER_CANCELLED    
                    print "Sub Order Status " + str(status)                      
                    
                    trackingDetailsActive = orderTrackingDet.find("div",{"class":"c-tabs-content m-top active"})
                    print "Sub order tracking description " + trackingDetailsActive.text
            if self._saveToOrder(todict(merchantOrder)):
                resp['result'] = 'ORDER_CREATED'
            else:
                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                
            return resp 
        except:
            traceback.print_exc()
            resp['result'] = 'ORDER_NOT_CREATED'
            return resp 
            
    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, value in Store.OrderStatusMap.iteritems():
            if detailedStatus.lower() in value:
                return key
        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
    
    def flipkartOrderTracking(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        br = getBrowserObject()
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
       
        response = br.open(url)
        page = ungzipResponse(response, br)
       
        
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
        
        
        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})
        #fetching suborders details
        for subOrder in mainOrder:
            subOrd = subOrder.findAll("li")
           
            subOrd.pop(len(subOrd)-1)
            for productInfo in subOrd:
                status=-1    
                #To track shipping details         
                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})
                for trackingStatus in n:
                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                    if "approveDetails-complete" in str(tr):
                        print "Here"
                        if "processingDetails-complete" in str(tr):
                            print "There"
                            if "shippingDetails-complete" in str(tr):
                                print "Share"
                                if "delivery-complete" in str(tr):
                                    print "Last"
                                    status = 3
                                else:     
                                    status = 2
                            else:
                                status = 1 
                        else: 
                            status = 0     
                    if "dead" in str(tr):
                        status = 4    
                print status                      
               
                
def _saveToOrderFlipkart(self, order):
        collection = self.db.merchantOrder
        order = collection.insert(order)


    
def hex_md5(password):
    m = hashlib.md5()
    print(m.digest())

def main():
    store = getStore(2)
    store.scrapeAffiliate(200)


        
        
        
def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj
    
if __name__ == '__main__':
        main()