Subversion Repositories SmartDukaan

'''
Created on Jan 15, 2015

@author: amit
'''
from BeautifulSoup import BeautifulSoup
from datetime import datetime, date, timedelta
from dtr import main
from dtr.dao import AffiliateInfo, Order, SubOrder, FlipkartAffiliateInfo
from dtr.main import getBrowserObject, getStore, ParseException, ungzipResponse, \
    Store as MStore, sourceMap, tprint
from dtr.reports.affiliatereco import getSkuData
from dtr.storage.DataService import Clicks, Users, FlipkartOrders
from dtr.utils.utils import fetchResponseUsingProxy
from elixir import *
from pprint import pprint
from pymongo.mongo_client import MongoClient
import hashlib
import importlib
import json
import mechanize
import pymongo
import re
import traceback
import urllib

USERNAME='saholic1@gmail.com'
PASSWORD='spice@2020'
ORDER_TRACK_URL='https://m.flipkart.com/order_details'
AFFILIATE_URL='https://www.flipkart.com/affiliate/login'
AFFILIATE_LOGIN_URL='https://www.flipkart.com/affiliate/a_login'
AFF_REPORT_URL='http://www.flipkart.com/affiliate/reports/ordersReport?order_status_filter=%s&startdate=%s&enddate=%s&page=%s'
AFF_STATUS_CANCELLED='cancelled'
AFF_STATUS_APPROVED='approved'
AFF_STATUS_DISAPPROVED='disapproved'
AFF_STATUS_PENDING='pending'
categoryMap = {3:"Mobiles", 5:"Tablets"}


class Store(MStore):
    OrderStatusMap = {
                      main.Store.ORDER_PLACED : ['approval', 'processing', 'shipping'],
                      main.Store.ORDER_DELIVERED : ['your item has been delivered'],
                      main.Store.ORDER_SHIPPED : ['in transit', 'shipment yet to be delivered'],
                      main.Store.ORDER_CANCELLED : ['shipment is returned', 'your item has been returned', 'your shipment has been cancelled', 'your shipment has been cancelled.']
                      
                      }
    def __init__(self,store_id):
        client = MongoClient('mongodb://localhost:27017/')
        self.db = client.dtr
        super(Store, self).__init__(store_id)

    def getName(self):
        return "flipkart"
    
    def scrapeStoreOrders(self,):
        orders = self._getActiveOrders()
        for order in orders:
            print "Order", self.store_name, order['orderId']
            try:
                closed = True
                url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', order['orderSuccessUrl'],re.IGNORECASE)[0]
                page = fetchResponseUsingProxy(url)
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
                
                sections = soup.findAll("div", {"class":"ui-app-card-body"})
                sections.pop(1)
        
                mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
                fkSubOrders = mainOrder.findAll("li")
                
                #remove unwanted list
                fkSubOrders.pop(-1)
                
                
                bulk = self.db.merchantOrder.initialize_ordered_bulk_op()
                #fetching suborders details
                for subOrder in fkSubOrders:
                    updateMap = {}
                    y = subOrder.find("header").findAll("span")
                    for suborderId in y:
                        if "value emp" in str(suborderId):
                            merchantSubOrderId = suborderId.text
                            break
                    ul = subOrder.find("ul")
                    if ul is None:
                        ul = subOrder.findAll("div", recursive=False)[0].div.div
                    orderItems = ul.findAll("div", recursive=False)
                    i=0
                    for orderItem in orderItems:
                        closedStatus = False
                        merchantSubOrderId = merchantSubOrderId + str(i)
                        subOrder =  self._isSubOrderActive(order, merchantSubOrderId)
                        if subOrder is None:
                            break
                        elif subOrder['closed']:
                            break
                        findMap = {"orderId": order['orderId'], "subOrders.merchantSubOrderId": merchantSubOrderId}
                        divs = orderItem.findAll('div', recursive=False)
                        orderTracking = divs[2]
                        orderTrackingDetDiv = divs[3].find('div',{'class':'c-tabs-content m-top active'})
                        orderTrackingDet = orderTrackingDetDiv.find('div',{'class':re.compile('tracking-remark')}).text
                        updateMap["subOrders.$.detailedStatus"] = orderTrackingDet
                        status = self._getStatusFromDetailedStatus(orderTrackingDet)
                        tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                        
                        cashbackStatus = subOrder.get("cashBackStatus") 
                        if "approveDetails-complete" in str(tr):
                            if "processingDetails-complete" in str(tr):
                                if "shippingDetails-complete" in str(tr):
                                    if "delivery-complete" in str(tr):
                                        status = MStore.ORDER_DELIVERED
                                        if cashbackStatus == Store.CB_PENDING:
                                            cashbackStatus = Store.CB_APPROVED
                                        closedStatus = True
                                    else:     
                                        status = MStore.ORDER_SHIPPED
                                        courierTrackAnchor = orderTrackingDetDiv.find('a')
                                        trackingUrl = courierTrackAnchor("href")
                                        trackingText = courierTrackAnchor.text
                                        courierArr = trackingText.split(' : ')
                                        updateMap["subOrders.$.trackingUrl"] = trackingUrl
                                        updateMap["subOrders.$.trackingNumber"] = courierArr[1]
                                        updateMap["subOrders.$.courierName"] = courierArr[0]
                                else:
                                    status = MStore.ORDER_CANCELLED
                                    closedStatus = True
                                    if cashbackStatus == Store.CB_PENDING:
                                        cashbackStatus = Store.CB_CANCELLED
                            else: 
                                status = MStore.ORDER_PLACED
                        elif str(tr) in ["approveDetails-ongoing"]:
                            status=MStore.ORDER_PLACED             
                        if "dead" in str(tr) or status==MStore.ORDER_CANCELLED:
                            status = MStore.ORDER_CANCELLED
                            closedStatus = True
                            if cashbackStatus == Store.CB_PENDING:
                                cashbackStatus = Store.CB_CANCELLED
                        
                        updateMap["subOrders.$.cashBackStatus"] = cashbackStatus
                        updateMap["subOrders.$.status"] = status
                        updateMap["subOrders.$.closed"] = closedStatus
                        if closed:
                            closed = closedStatus
                        bulk.find(findMap).update({'$set' : updateMap})
                bulk.find({'orderId': order['orderId']}).update({'$set':{'closed': closed, 'parseError':False}})
                bulk.execute()            
            except:
                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())
                traceback.print_exc()
    def scrapeAffiliate(self, startDate=None, endDate=None):
        #get all yesterday's affiliate in pending or cancelled state and update to system
        offers = []
        br = getBrowserObject()
        br.open(AFFILIATE_URL)
        response = br.response()  # copy
        token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
        data = {'__FK':token,
                'email':'saholic1@gmail.com',
                'password':'e8aacf6fc1e3998186a4a8e56e428f66'
        }
        br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
        for delta in range(1,2):
            yester5date = date.today() - timedelta(delta)
            syester5date = yester5date.strftime('%Y-%m-%d')
            for status in [AFF_STATUS_PENDING, AFF_STATUS_CANCELLED, AFF_STATUS_DISAPPROVED]:
                hasPagination=True
                pagination = 0
                while hasPagination:
                    pagination = pagination + 1
                    print "pagination", pagination
                    try:
                        br.open(AFF_REPORT_URL % (status, syester5date, syester5date, pagination))
                    except:
                        tprint("Could not fetch data for Status %s and date %s"%(status, syester5date))
                    page = ungzipResponse(br.response())
                    soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
                    soup.table
                    try:
                        tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]
                    except:
                        break
                    trElements = tableElement.findAll('tr')
                    trElements.pop(0)
                        
                    for trElement in trElements:
                        tdElements = trElement.findAll('td')
                        if tdElements[0].text.strip()=='No Products to display!':
                            hasPagination = False
                            continue
                        productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]
                        quantity =int(tdElements[3].text)
                        price = int(float(tdElements[2].text.strip().replace(",","")))
                        payOut = int(float(tdElements[6].text.strip().replace(",","")))
                        saleAmount = int(float(tdElements[4].text.strip().replace(",","")))
                        subTagId = tdElements[7].text.strip()
                        category = tdElements[1].text.strip()
                        affiliateInfo = FlipkartAffiliateInfo(subTagId, syester5date, productCode, price, quantity, saleAmount, payOut, status, category)
                        affiliateInfo.productTitle =  tdElements[0].find('a').text
                        #updateMap['subOrders.$.unitPrice'] = price
                        #updateMap['subOrders.$.cashBackAmount'], updateMap['subOrders.$.cashBacPercentage'] = self.getCashbackAmount(productCode, price)
                        offers.append(affiliateInfo)
        self._saveToAffiliate(offers)
        
        yester5date = date.today() - timedelta(1)
        syester5date = yester5date.strftime('%Y-%m-%d')
        self.db.flipkartOrderAffiliateInfo.find({"saleDate":syester5date})
        for order in self.db.flipkartOrderAffiliateInfo.find({"saleDate":syester5date}):
            userId = None
            subTagId = None
            email = None
            subTagId = order.get("subTagId")
            if subTagId:
                click = session.query(Clicks).filter_by(tag = subTagId).first()
                if click is not None:
                    userId= click.user_id 
                    user = session.query(Users.email).filter_by(id = userId).first()
                    if user is not None:
                        email = user.email
            
            flipkartOrder = FlipkartOrders()
            flipkartOrder.user_id = userId
            flipkartOrder.identifier = order.get("identifier")
            flipkartOrder.email = email
            flipkartOrder.subtagId = order.get("subTagId")
            flipkartOrder.created = datetime.strptime(order.get("saleDate"), "%Y-%m-%d")
            flipkartOrder.status = order.get("conversionStatus")
            flipkartOrder.title = order.get("productTitle")
            flipkartOrder.price = order.get("price")
            flipkartOrder.quantity = order.get("quantity")
            flipkartOrder.productCode = order.get("productCode")
            skuData = getSkuData(2, order.get("productCode"))
            if skuData is not None:
                flipkartOrder.catalogId = skuData.get("skuBundleId")
                flipkartOrder.brand = skuData.get("brand")
                flipkartOrder.model = skuData.get("model_name")
                flipkartOrder.category = categoryMap.get(skuData.get("category_id"))
                flipkartOrder.title =skuData.get("source_product_name")
                        
        session.commit()
    
    def _saveToAffiliate(self, offers):
        collection = self.db.flipkartOrderAffiliateInfo
        mcollection = self.db.merchantOrder
        for offer in offers:
            result1 = collection.update({"subTagId":offer.subTagId, "price":offer.price, "quantity":offer.quantity, "saleDate":offer.saleDate, "productCode":offer.productCode},{"$set":todict(offer)}, upsert=True)
            if result1.get("upserted") is not None:
                result2 = mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True, "subOrders.productCode":offer.productCode}, {"$set":{"subOrders.$.missingAff":False}}, multi=True)
                if result2.get("updatedExisting") is True:
                    mcollection.update({"_id":result1.get("upserted")}, {"missingOrders":False})
                
            
            
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        resp= {}
        try:
            br = getBrowserObject()
            url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
           
            response = br.open(url)
            page = ungzipResponse(response)
            
            merchantOrderId = re.findall('reference_id=(.*?)&', orderSuccessUrl,re.IGNORECASE)[0]
            print merchantOrderId
            
            soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
            
            sections = soup.findAll("div", {"class":"ui-app-card-body"})
            sections.pop(1)
    
            merchantOrder = Order(orderId, userId, subTagId, self.store_id, orderSuccessUrl)
            merchantOrder.orderTrackingUrl = url
            for data in sections:
                name = data.findAll("span")
                i=0
                while i< len(name):
                    if "key" in str(name[i]):
                        if "Grand Total" in str(name[i]):
                            #Total Amount
                            merchantOrder.paidAmount = int(re.findall(r'\d+', name[i+1].text)[0])
                        elif "Order Date" in str(name[i]):
                            merchantOrder.placedOn = name[i+1].text 
                    i=i+1    
            
            merchantOrder.merchantOrderId = merchantOrderId
    
            mainOrder = soup.find("ul",{"class":"m-bottom p-cart"})
            fkSubOrders = mainOrder.findAll("li")
            #remove unwanted list
            fkSubOrders.pop(-1)
            subOrders = []
            merchantOrder.subOrders = subOrders
    
            #fetching suborders details
            for subOrder in fkSubOrders:
                y = subOrder.find("header").findAll("span")
                for suborderId in y:
                    if "value emp" in str(suborderId):
                        merchantSubOrderId = suborderId.text
                        break
                orderItems = subOrder.find("ul").findAll("div", recursive=False)
                i=0
                for orderItem in orderItems:
                    merchantOrder.placedOn
                    merchantSubOrderId = merchantSubOrderId + str(i)
                    divs = orderItem.findAll('div', recursive=False)
                    content = divs[0]
                    orderTracking = divs[2]
                    orderTrackingDet = divs[3]
                    
                    
                    imgUrl = content.find('img')['src']
                    lineDet = content.find("div",{"class":"product-info"})
                    productTitle = lineDet.find("a",{"class":"product-title"}).text
                    productUrl =  str(lineDet.find("a")['href'])
    
                    start='pid='
                    s=str(lineDet.find("a")['href'])
                    productCode = re.findall(re.escape(start)+"(.*)",s)[0].strip()
                    mname = lineDet.findAll("span")
                    k=0
                    while k<len(mname):
                        if "note" in str(mname[k]):
                            if "Color:" in str(mname[k]):
                                productTitle = productTitle + " " + mname[k+1].text
                                
                            elif "Qty:" in str(mname[k]):
                                quantity = int(mname[k+1].text)
                            elif "Subtotal:" in str(mname[k]):
                                amountPaid = int(re.findall(r'\d+', mname[k+1].text)[0])
                            elif "Delivery:" in str(mname[k]):
                                #Delivery Date for sub order
                                print "Delivery Date " +mname[k+1].text
                        k=k+1
                    merchantsubOrder = SubOrder(productTitle, productUrl, merchantOrder.placedOn, amountPaid,MStore.ORDER_PLACED, quantity)
                    merchantsubOrder.imgUrl  = imgUrl
                    merchantsubOrder.productCode = productCode
                    merchantsubOrder.merchantSubOrderId = merchantSubOrderId
                    print "productCode", productCode
                    print "amountPaid", amountPaid
                    cashbackAmount, cashbackPercent = self.getCashbackAmount(productCode, amountPaid)
                    cashbackStatus = Store.CB_PENDING
                    if cashbackAmount <= 0:
                        cashbackStatus = Store.CB_NA
                    merchantsubOrder.cashBackAmount = cashbackAmount
                    merchantsubOrder.cashBackStatus = cashbackStatus      
                    merchantsubOrder.cashBackPercentage = cashbackPercent      
                    
                    subOrders.append(merchantsubOrder)
                    #To track shipping details         
                    status=-1    
                    tr = orderTracking.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                    if "approveDetails-complete" in str(tr):
                        if "processingDetails-complete" in str(tr):
                            if "shippingDetails-complete" in str(tr):
                                if "delivery-complete" in str(tr):
                                    status = MStore.ORDER_DELIVERED
                                else:     
                                    status = MStore.ORDER_SHIPPED
                                    orderTracking.find('div', {'class':'tracking-remark active'})
                            else:
                                status = MStore.ORDER_PLACED
                        else: 
                            status = MStore.ORDER_PLACED
                    elif str(tr) in ["approveDetails-ongoing"]:
                        status=MStore.ORDER_PLACED             
                    if "dead" in str(tr) or "shippingDetails-returnOngoing" in str(tr) or "shippingDetails-return" in str(tr):
                        status = MStore.ORDER_CANCELLED    
                    print "Sub Order Status " + str(status)                      
                    
                    trackingDetailsActive = orderTrackingDet.find("div",{"class":"c-tabs-content m-top active"})
                    print "Sub order tracking description " + trackingDetailsActive.text
            if self._saveToOrder(todict(merchantOrder)):
                resp['result'] = 'ORDER_CREATED'
            else:
                resp['result'] = 'ORDER_ALREADY_CREATED_IGNORED'
                
            return resp 
        except:
            traceback.print_exc()
            resp['result'] = 'ORDER_NOT_CREATED'
            return resp 
            
    def _getStatusFromDetailedStatus(self, detailedStatus):
        for key, value in Store.OrderStatusMap.iteritems():
            if detailedStatus.lower() in value:
                return key
        print "Detailed Status need to be mapped", detailedStatus, "Store:", self.store_name
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
    
    def flipkartOrderTracking(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
        br = getBrowserObject()
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
       
        response = br.open(url)
        page = ungzipResponse(response, br)
       
        
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
        
        
        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})
        #fetching suborders details
        for subOrder in mainOrder:
            subOrd = subOrder.findAll("li")
           
            subOrd.pop(len(subOrd)-1)
            for productInfo in subOrd:
                status=-1    
                #To track shipping details         
                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})
                for trackingStatus in n:
                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
                    if "approveDetails-complete" in str(tr):
                        print "Here"
                        if "processingDetails-complete" in str(tr):
                            print "There"
                            if "shippingDetails-complete" in str(tr):
                                print "Share"
                                if "delivery-complete" in str(tr):
                                    print "Last"
                                    status = 3
                                else:     
                                    status = 2
                            else:
                                status = 1 
                        else: 
                            status = 0     
                    if "dead" in str(tr):
                        status = 4    
                print status                      
               
                
def _saveToOrderFlipkart(self, order):
        collection = self.db.merchantOrder
        order = collection.insert(order)


    
def hex_md5(password):
    m = hashlib.md5()
    print(m.digest())

def main():
  
    store = getStore(2)
    store.scrapeAffiliate()
    #store.scrapeStoreOrders()
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')
    #hex_md5('spice@2020')
    #store.getCashbackAmount('MOBDVHC6XKKPZ3GZ', 5999)


if __name__ == '__main__':
        main()
        
        
    
        
def todict(obj, classkey=None):
    if isinstance(obj, dict):
        data = {}
        for (k, v) in obj.items():
            data[k] = todict(v, classkey)
        return data
    elif hasattr(obj, "_ast"):
        return todict(obj._ast())
    elif hasattr(obj, "__iter__"):
        return [todict(v, classkey) for v in obj]
    elif hasattr(obj, "__dict__"):
        data = dict([(key, todict(value, classkey)) 
            for key, value in obj.__dict__.iteritems() 
            if not callable(value) and not key.startswith('_')])
        if classkey is not None and hasattr(obj, "__class__"):
            data[classkey] = obj.__class__.__name__
        return data
    else:
        return obj