Subversion Repositories SmartDukaan

Rev

Rev 16706 | Rev 17240 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 16706 Rev 17238
Line 14... Line 14...
14
from dtr.utils import utils
14
from dtr.utils import utils
15
from dtr.utils.utils import fetchResponseUsingProxy
15
from dtr.utils.utils import fetchResponseUsingProxy
16
from elixir import *
16
from elixir import *
17
from pprint import pprint
17
from pprint import pprint
18
from pymongo.mongo_client import MongoClient
18
from pymongo.mongo_client import MongoClient
-
 
19
import StringIO
-
 
20
import csv
19
import hashlib
21
import hashlib
20
import importlib
22
import importlib
21
import json
23
import json
22
import mechanize
24
import mechanize
23
import pymongo
25
import pymongo
24
import re
26
import re
25
import traceback
27
import traceback
26
import urllib
28
import urllib
-
 
29
import zipfile
27
 
30
import urllib2
28
USERNAME='saholic1@gmail.com'
31
USERNAME='saholic1@gmail.com'
29
PASSWORD='spice@2020'
32
PASSWORD='spice@2020'
30
ORDER_TRACK_URL='https://m.flipkart.com/order_details'
33
ORDER_TRACK_URL='https://m.flipkart.com/order_details'
31
AFFILIATE_URL='https://www.flipkart.com/affiliate/login'
34
AFFILIATE_URL='https://affiliate.flipkart.com/'
32
AFFILIATE_LOGIN_URL='https://www.flipkart.com/affiliate/a_login'
35
AFFILIATE_LOGIN_URL='https://affiliate.flipkart.com/a_login'
33
AFF_REPORT_URL='http://www.flipkart.com/affiliate/reports/ordersReport?order_status_filter=%s&startdate=%s&enddate=%s&page=%s'
36
AFF_REPORT_URL = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"
-
 
37
AFF_DOWNLOAD_URL="https://affiliate.flipkart.com/downloads/file?id=%s"
34
AFF_STATUS_CANCELLED='cancelled'
38
AFF_STATUS_CANCELLED='cancelled'
35
AFF_STATUS_APPROVED='approved'
39
AFF_STATUS_APPROVED='approved'
36
AFF_STATUS_DISAPPROVED='disapproved'
40
AFF_STATUS_DISAPPROVED='disapproved'
37
AFF_STATUS_PENDING='pending'
41
AFF_STATUS_PENDING='pending'
38
statuses = [AFF_STATUS_CANCELLED,AFF_STATUS_APPROVED,AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]
42
statuses = [AFF_STATUS_CANCELLED, AFF_STATUS_APPROVED, AFF_STATUS_DISAPPROVED,AFF_STATUS_PENDING ]
39
categoryMap = {3:"Mobiles", 5:"Tablets"}
43
categoryMap = {3:"Mobiles", 5:"Tablets"}
40
 
44
 
41
 
45
 
42
class Store(MStore):
46
class Store(MStore):
43
    OrderStatusMap = {
47
    OrderStatusMap = {
Line 53... Line 57...
53
        super(Store, self).__init__(store_id)
57
        super(Store, self).__init__(store_id)
54
 
58
 
55
    def getName(self):
59
    def getName(self):
56
        return "flipkart"
60
        return "flipkart"
57
    
61
    
-
 
62
    
-
 
63
    def requestDownload(self):
-
 
64
    #"https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%7B%22filter%22%3A%22approved%22%2C%22till%22%3A%222015-10-03%22%2C%22from%22%3A%222015-04-03%22%7D"
-
 
65
        requestReportUrl = "https://affiliate.flipkart.com/downloads/a_downloadRequest?type=OrdersReport&parameters=%s"
-
 
66
        br = getBrowserObject()
-
 
67
        br.set_debug_responses(True)
-
 
68
        br.open(AFFILIATE_URL)
-
 
69
        response = br.response()  # copy
-
 
70
        #token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]
-
 
71
        data = {
-
 
72
                    'j_username':'saholic1@gmail.com',
-
 
73
                    'j_password':'spice@2020'
-
 
74
            }
-
 
75
        print utils.ungzipResponse(br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data)))
-
 
76
        till = datetime.strftime(date.today(),"%Y-%m-%d")
-
 
77
        start = datetime.strftime(date.today() - timedelta(4), "%Y-%m-%d")
-
 
78
        #target = open("dowloadreportids", 'w')
-
 
79
        #target.truncate()
-
 
80
        for status in statuses:
-
 
81
            #try:
-
 
82
                data = {"till":till, "from":start, "filter":status}
-
 
83
                print json.dumps(data)
-
 
84
                request = requestReportUrl%(urllib.quote_plus(json.dumps(data).replace(" ", "")))
-
 
85
                print request
-
 
86
                response = utils.ungzipResponse(br.open(request))
-
 
87
                response = json.loads(response)
-
 
88
                print response
-
 
89
                if(response['status']=="OK"):
-
 
90
                    self.db.flipkartdownloadids.save(response)
-
 
91
                else:
-
 
92
                    utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod")
-
 
93
                    #break 
-
 
94
            #except:
-
 
95
                utils.sendmail(['amit.gupta@shop2020.in'], '', "Could not get request data for Flipkart Affiliate downlaod due to Internal Server Error")
-
 
96
                #break 
-
 
97
                
-
 
98
    
58
    def scrapeStoreOrders(self,):
99
    def scrapeStoreOrders(self,):
59
        orders = self._getActiveOrders()
100
        orders = self._getActiveOrders()
60
        for order in orders:
101
        for order in orders:
61
            print "Order", self.store_name, order['orderId']
102
            print "Order", self.store_name, order['orderId']
62
            try:
103
            try:
Line 149... Line 190...
149
                bulk.execute()            
190
                bulk.execute()            
150
            except:
191
            except:
151
                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
192
                self.db.merchantOrder.update({"orderId":order['orderId']}, {"$set":{"parseError":True}})
152
                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())
193
                tprint("Could not update " + str(order['orderId']) + ' for store ' + self.getName())
153
                traceback.print_exc()
194
                traceback.print_exc()
-
 
195
    
154
    def scrapeAffiliate(self, startDate=None, endDate=None):
196
    def scrapeAffiliate(self, deltaDays=0):
155
        if startDate==None:
197
        if deltaDays is None:
156
            startDate=2
198
            deltaDays=0
157
        #get all yesterday's affiliate in pending or cancelled state and update to system
-
 
158
        offers = []
-
 
159
        br = getBrowserObject()
-
 
160
        br.open(AFFILIATE_URL)
-
 
161
        response = br.response()  # copy
-
 
162
        token =  re.findall('window.__FK = "(.*?)"', ungzipResponse(response), re.IGNORECASE)[0]
-
 
163
        data = {'__FK':token,
-
 
164
                'email':'saholic1@gmail.com',
-
 
165
                'password':'e8aacf6fc1e3998186a4a8e56e428f66'
-
 
166
        }
-
 
167
        br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
-
 
168
        for delta in range(1,startDate):
-
 
169
            yester5date = date.today() - timedelta(delta)
199
        endDate=date.today() - timedelta(days=1)
170
            syester5date = yester5date.strftime('%Y-%m-%d')
-
 
171
            #print "syester5date", syester5date
-
 
172
            for status in [AFF_STATUS_PENDING, AFF_STATUS_CANCELLED, AFF_STATUS_DISAPPROVED,AFF_STATUS_APPROVED]:
-
 
173
                #print "status", status
-
 
174
                hasPagination=True
-
 
175
                pagination = 0
-
 
176
                while hasPagination:
-
 
177
                    pagination = pagination + 1
-
 
178
                    try:
-
 
179
                        br.open(AFF_REPORT_URL % (status, syester5date, syester5date, pagination))
-
 
180
                    except:
-
 
181
                        tprint("Could not fetch data for Status %s and date %s and pagination %s"%(status, syester5date, pagination))
-
 
182
                        continue
-
 
183
                    page = ungzipResponse(br.response())
-
 
184
                    soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
-
 
185
                    #print "Soup", soup.table
-
 
186
                    try:
-
 
187
                        tableElement = soup.findAll('table', {'class':'report-table fixtable'})[1]
-
 
188
                    except:
-
 
189
                        break
-
 
190
                    trElements = tableElement.findAll('tr')
-
 
191
                    trElements.pop(0)
-
 
192
                    for trElement in trElements:
-
 
193
                        tdElements = trElement.findAll('td')
-
 
194
                        if tdElements[0].text.strip()=='No Products to display!':
-
 
195
                            hasPagination = False
-
 
196
                            continue
-
 
197
                        productCode = re.findall(r'pid=(.*)$', tdElements[0].find('a')['href'])[0]
-
 
198
                        quantity =int(tdElements[3].text)
-
 
199
                        price = int(float(tdElements[2].text.strip().replace(",","")))
-
 
200
                        payOut = int(float(tdElements[6].text.strip().replace(",","")))
-
 
201
                        saleAmount = int(float(tdElements[4].text.strip().replace(",","")))
-
 
202
                        subTagId = tdElements[7].text.strip()
-
 
203
                        category = tdElements[1].text.strip()
200
        startDate = endDate - timedelta(days=deltaDays)
204
                        affiliateInfo = FlipkartAffiliateInfo(subTagId, syester5date, productCode, price, quantity, saleAmount, payOut, status, category)
-
 
205
                        affiliateInfo.productTitle =  tdElements[0].find('a').text
-
 
206
                        #updateMap['subOrders.$.unitPrice'] = price
-
 
207
                        #updateMap['subOrders.$.cashBackAmount'], updateMap['subOrders.$.cashBacPercentage'] = self.getCashbackAmount(productCode, price)
-
 
208
                        offers.append(affiliateInfo)
-
 
209
                self._saveToAffiliate(offers)
-
 
210
        
201
        
-
 
202
        endDate = datetime.strftime(endDate, "%Y-%m-%d")
-
 
203
        startDate = datetime.strftime(startDate, "%Y-%m-%d")
-
 
204
        url  = "https://affiliate-api.flipkart.net/affiliate/report/orders/detail/json?startDate=%s&endDate=%s&status=%s&offset=%s"
-
 
205
            
-
 
206
        sum = 0
211
        for delta in range(1,startDate):
207
        for status in statuses:
-
 
208
            offset=-1
-
 
209
            while True:
-
 
210
                offset += 1
212
            yester5date = date.today() - timedelta(delta)
211
                reqUlrl = url%(startDate, endDate, status, offset) 
213
            syester5date = yester5date.strftime('%Y-%m-%d')
212
                req = urllib2.Request(reqUlrl)
214
            self.db.flipkartOrderAffiliateInfo.find({"saleDate":syester5date})
213
                req.add_header('Fk-Affiliate-Id', 'saholic1g')
215
            for order in self.db.flipkartOrderAffiliateInfo.find({"saleDate":syester5date}):
214
                req.add_header('Fk-Affiliate-Token', 'a757444e260c46be8c4aeb20352246ac')
-
 
215
                resp = urllib2.urlopen(req)
-
 
216
                resString = json.loads(resp.read())
-
 
217
                orderList = resString["orderList"]
216
                userId = None
218
                if orderList:
217
                subTagId = None
219
                    for order in orderList:
218
                email = None
220
                        print order
-
 
221
                        order['sales'] = int(order['sales']['amount'])
-
 
222
                        order['tentativeCommission'] = int(order['tentativeCommission']['amount'])
-
 
223
                        if order['status']!='failed':
-
 
224
                             sum += order['tentativeCommission']
-
 
225
                        self.db.flipkartOrderAffiliateInfo1.save(order)
-
 
226
                        skuData = getSkuData(2, order.get("productCode"))
-
 
227
                        if skuData is not None:
-
 
228
                            order['catalogId'] = skuData.get("skuBundleId")
-
 
229
                            order['brand'] = skuData.get("brand")
-
 
230
                            order['model'] = skuData.get("model_name")
-
 
231
                            order['category'] = categoryMap.get(skuData.get("category_id"))
-
 
232
 
-
 
233
 
219
                subTagId = order.get("subTagId")
234
                        subTagId = order.get("affExtParam1")
-
 
235
                        userId = None
-
 
236
                        email = None
220
                if subTagId:
237
                        if subTagId:
221
                    click = session.query(Clicks).filter_by(tag = subTagId).first()
238
                            click = session.query(Clicks).filter_by(tag = subTagId).first()
222
                    if click is not None:
239
                            if click is not None:
223
                        userId= click.user_id 
240
                                userId= click.user_id 
224
                        user = session.query(Users.email).filter_by(id = userId).first()
241
                                user = session.query(Users.email).filter_by(id = userId).first()
225
                        if user is not None:
242
                                if user is not None:
226
                            email = user.email
243
                                    email = user.email
227
                
244
                        
228
                flipkartOrder = FlipkartOrders()
245
                        flipkartOrder = FlipkartOrders()
-
 
246
                        flipkartOrder.subtagId = subTagId
229
                flipkartOrder.user_id = userId
247
                        flipkartOrder.user_id = userId
230
                flipkartOrder.identifier = order.get("identifier")
248
                        flipkartOrder.identifier = order.get("identifier")
231
                flipkartOrder.email = email
249
                        flipkartOrder.email = email
232
                flipkartOrder.subtagId = order.get("subTagId")
-
 
233
                flipkartOrder.created = datetime.strptime(order.get("saleDate"), "%Y-%m-%d")
250
                        flipkartOrder.created = datetime.strptime(order.get("orderDate"), "%m-%d-%Y")
234
                flipkartOrder.status = order.get("conversionStatus")
251
                        flipkartOrder.status = order.get("status")
235
                flipkartOrder.title = order.get("productTitle")
252
                        flipkartOrder.title = order.get("title")
236
                flipkartOrder.price = order.get("price")
253
                        flipkartOrder.price = order.get("price")
237
                flipkartOrder.quantity = order.get("quantity")
254
                        flipkartOrder.quantity = order.get("quantity")
238
                flipkartOrder.productCode = order.get("productCode")
255
                        flipkartOrder.productCode = order.get("productId")
-
 
256
                        flipkartOrder.affiliateOrderItemId = order.get("affiliateOrderItemId")
-
 
257
                        flipkartOrder.payOut = order['tentativeCommission']
-
 
258
                        flipkartOrder.payOutPercentage = order['commissionRate']
239
                skuData = getSkuData(2, order.get("productCode"))
259
                        skuData = getSkuData(2, order.get("productId"))
240
                if skuData is not None:
260
                        if skuData is not None:
241
                    flipkartOrder.catalogId = skuData.get("skuBundleId")
261
                            flipkartOrder.catalogId = skuData.get("skuBundleId")
242
                    flipkartOrder.brand = skuData.get("brand")
262
                            flipkartOrder.brand = skuData.get("brand")
243
                    flipkartOrder.model = skuData.get("model_name")
263
                            flipkartOrder.model = skuData.get("model_name")
244
                    flipkartOrder.category = categoryMap.get(skuData.get("category_id"))
264
                            flipkartOrder.category = categoryMap.get(skuData.get("category_id"))
245
                    flipkartOrder.title =skuData.get("source_product_name")
265
                            flipkartOrder.title =skuData.get("source_product_name")
246
                            
266
                                    
247
            session.commit()
267
                    session.commit()
-
 
268
                else:
-
 
269
                    break
248
    
270
    
249
    def _saveToAffiliate(self, offers):
271
    def _saveToAffiliate(self, offers):
250
        collection = self.db.flipkartOrderAffiliateInfo
272
        collection = self.db.flipkartOrderAffiliateInfo1
-
 
273
        count=0
-
 
274
        for row in offers:
-
 
275
            if count==0:
-
 
276
                count += 1
-
 
277
                continue
251
        mcollection = self.db.merchantOrder
278
            offer = self.covertToObj(row)
252
        for offer in offers:
279
            collection.save(offer)
-
 
280
    
-
 
281
    def covertToObj(self,offer):
253
            result1 = collection.update({"subTagId":offer.subTagId, "price":offer.price, "quantity":offer.quantity, "saleDate":offer.saleDate, "productCode":offer.productCode},{"$set":todict(offer)}, upsert=True)
282
        affiliateorderitemid,title,productid,category,quantity,sales,price,commissionrate,tentativecommission,status,orderdate,saleschannel,customertype,affextparam1, affextparam2  = offer
-
 
283
        saleMap = {
254
            if result1.get("upserted") is not None:
284
               "affiliateorderitemid":affiliateorderitemid,
-
 
285
                "title":title,
-
 
286
                "productid":productid,
-
 
287
                "category":category,
-
 
288
                "quantity":quantity,
-
 
289
                "saleAmount":sales,
-
 
290
                "price":price,
255
                result2 = mcollection.update({"subTagId":offer.subTagId, "storeId":self.store_id, "subOrders.missingAff":True, "subOrders.productCode":offer.productCode}, {"$set":{"subOrders.$.missingAff":False}}, multi=True)
291
                "commissionrate":commissionrate,
256
                if result2.get("updatedExisting") is True:
292
                "payOut":tentativecommission,
257
                    mcollection.update({"_id":result1.get("upserted")}, {"missingOrders":False})
293
                "conversionStatus":status,
-
 
294
                "saleDate":orderdate,
-
 
295
                "saleschannel":saleschannel,
-
 
296
                "customertype":customertype,
-
 
297
                "affextparam1":affextparam1,
-
 
298
                "_id":affiliateorderitemid
258
                
299
        }
-
 
300
        return saleMap
259
            
301
            
260
            
302
            
261
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
303
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
262
        resp= {}
304
        resp= {}
263
        try:
305
        try:
Line 447... Line 489...
447
def hex_md5(password):
489
def hex_md5(password):
448
    m = hashlib.md5()
490
    m = hashlib.md5()
449
    print(m.digest())
491
    print(m.digest())
450
 
492
 
451
def main():
493
def main():
452
    #requestDownload()
-
 
453
    
-
 
454
    
-
 
455
    
-
 
456
 
-
 
457
    store = getStore(2)
494
    store = getStore(2)
458
    #store.scrapeAffiliate(60)
495
    store.scrapeAffiliate(200)
459
    store.scrapeAffiliate()
-
 
460
    #store.scrapeStoreOrders()
-
 
461
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
-
 
462
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
-
 
463
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
-
 
464
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
-
 
465
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
-
 
466
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')
-
 
467
    #hex_md5('spice@2020')
-
 
468
    #store.getCashbackAmount('MOBDVHC6XKKPZ3GZ', 5999)
-
 
469
 
496
 
470
 
497
 
471
        
498
        
472
        
499
        
473
def requestDownload():
-
 
474
    requestReportUrl = "http://www.flipkart.com/affiliate/download/request"
-
 
475
    br = getBrowserObject()
-
 
476
    br.set_debug_responses(True)
-
 
477
    br.open(AFFILIATE_URL)
-
 
478
    response = br.response()  # copy
-
 
479
    token =  re.findall('window.__FK = "(.*?)"', utils.ungzipResponse(response), re.IGNORECASE)[0]
-
 
480
    data = {'__FK':token,
-
 
481
            'email':'saholic1@gmail.com',
-
 
482
            'password':'e8aacf6fc1e3998186a4a8e56e428f66'
-
 
483
    }
-
 
484
    br.open(AFFILIATE_LOGIN_URL, urllib.urlencode(data))
-
 
485
    till = date.today()
-
 
486
    start = date.today() - timedelta(2)
-
 
487
    target = open("dowloadreportids", 'w')
-
 
488
    target.truncate() 
-
 
489
    for status in statuses:
-
 
490
        data = getRequestDownloadData(token, start, till, status)
-
 
491
        response = utils.ungzipResponse(br.open(requestReportUrl, urllib.urlencode(data)))
-
 
492
        target.write(response)
-
 
493
        
-
 
494
    target.close()
-
 
495
    
-
 
496
    
-
 
497
def getRequestDownloadData(token, start, till, status):
-
 
498
    return (
-
 
499
            ('__FK',token),
-
 
500
            ('parameters[from]', start),
-
 
501
            ('parameters[till]', till),
-
 
502
            ('parameters[filter]', status),
-
 
503
            ('type','OrdersReport'),
-
 
504
            )
-
 
505
    
-
 
506
        
500
        
507
def todict(obj, classkey=None):
501
def todict(obj, classkey=None):
508
    if isinstance(obj, dict):
502
    if isinstance(obj, dict):
509
        data = {}
503
        data = {}
510
        for (k, v) in obj.items():
504
        for (k, v) in obj.items():