Subversion Repositories SmartDukaan

Rev

Rev 14624 | Rev 14651 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14624 Rev 14650
Line 4... Line 4...
4 
 
 4 
 
5 
@author: amit
 5 
@author: amit
6 
'''
 6 
'''
7 
from base64 import encode
 7 
from base64 import encode
8 
from bs4 import BeautifulSoup
 8 
from bs4 import BeautifulSoup
9 
from datetime import datetime
 9 
from datetime import datetime, timedelta
10 
from dtr.dao import Order, SubOrder
 10 
from dtr.dao import Order, SubOrder, AmazonAffiliateInfo
11 
from dtr.main import getStore, Store as MStore, ParseException, getBrowserObject, \
 11 
from dtr.main import getStore, Store as MStore, ParseException, getBrowserObject, \
12 
    ungzipResponse, tprint
 12 
    ungzipResponse, tprint
13 
from dtr.sources.flipkart import todict
 13 
from dtr.sources.flipkart import todict, AFF_REPORT_URL
14 
from paramiko import sftp
 14 
from paramiko import sftp
15 
from paramiko.client import SSHClient
 15 
from paramiko.client import SSHClient
16 
from paramiko.sftp_client import SFTPClient
 16 
from paramiko.sftp_client import SFTPClient
- 
 
 17 
import base64
- 
 
 18 
import gzip
17 
import mechanize
 19 
import mechanize
18 
import os.path
 20 
import os.path
19 
import paramiko
 21 
import paramiko
20 
import re
 22 
import re
21 
import time
 23 
import time
22 
import traceback
 24 
import traceback
- 
 
 25 
import urllib2
23 
 
 26 
 
24 
ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'
 27 
ORDER_REDIRECT_URL = 'https://www.amazon.in/gp/css/summary/edit.html?orderID=%s'
25 
ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'
 28 
ORDER_SUCCESS_URL = 'https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html'
26 
THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'
 29 
THANKYOU_URL = 'https://www.amazon.in/gp/buy/thankyou/handlers/display.html'
27 
AMAZON_AFF_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/listReports'
 30 
AMAZON_AFF_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/listReports'
- 
 
 31 
AMAZON_AFF_FILE_URL = 'https://assoc-datafeeds-eu.amazon.com/datafeed/getReport?filename=saholic-21-orders-report-%s.tsv.gz'
28 
class Store(MStore):
 32 
class Store(MStore):
29
33
30 
    orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],
 34 
    orderStatusRegexMap = { MStore.ORDER_PLACED : ['ordered from', 'not yet dispatched','dispatching now', 'preparing for dispatch'],
31 
                            MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],
 35 
                            MStore.ORDER_SHIPPED : ['dispatched on','dispatched', 'on the way', 'out for delivery', 'Out for delivery'],
32 
                            MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],
 36 
                            MStore.ORDER_CANCELLED : ['return complete', 'refunded', 'cancelled'],
Line 417... Line 421...
417
421
418 
        print "Detailed Status need to be mapped"
 422 
        print "Detailed Status need to be mapped"
419 
        print "Found new order status", detailedStatus
 423 
        print "Found new order status", detailedStatus
420 
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
 424 
        raise ParseException("_getStatusFromDetailedStatus", "Found new order status" + detailedStatus)
421 
    def scrapeAffiliate(self, startDate=None, endDate=None):
 425 
    def scrapeAffiliate(self, startDate=None, endDate=None):
- 
 
 426 
        br = getBrowserObject()
- 
 
 427 
        br.add_password('https://assoc-datafeeds-eu.amazon.com', 'Saholic', 'Fnubyvp')
- 
 
 428 
        url = AMAZON_AFF_URL
- 
 
 429 
        response = br.open(url)
- 
 
 430 
        #get data for past 40 days and store it to mongo
- 
 
 431 
        dt = datetime.now()
- 
 
 432 
        dtprevious = dt - timedelta(days=41)
- 
 
 433 
        for i in xrange(40):
- 
 
 434 
            dat = dtprevious + timedelta(days=i)
- 
 
 435 
            url = AMAZON_AFF_FILE_URL%(datetime.strftime(dat, "%Y%m%d"))
- 
 
 436 
            response = br.open(url)
- 
 
 437 
            page = gzip.GzipFile(fileobj=response, mode='rb').read()
422 
        pass
 438 
            j=-1
- 
 
 439 
            for row in page.split("\n"):
- 
 
 440 
                j += 1
- 
 
 441 
                if j== 0 or j==1:
- 
 
 442 
                    continue
- 
 
 443 
                fields = row.split("\t")
- 
 
 444 
                if len(fields)>1:
- 
 
 445 
                    print fields
- 
 
 446 
                    amazonAffiliate = AmazonAffiliateInfo(fields[0], fields[1], fields[2], fields[3], fields[4], fields[5], fields[6], fields[7], fields[8], fields[9])
- 
 
 447 
                    print amazonAffiliate
- 
 
 448 
                    self.db.amazonAffiliateInfo.insert(todict(amazonAffiliate))
- 
 
 449 
                else:
- 
 
 450 
                    break
423
451
424
452
425 
    def parseTrackingUrl(self, br, trackingUrl):
 453 
    def parseTrackingUrl(self, br, trackingUrl):
426 
        subOrder = {}
 454 
        subOrder = {}
427 
        response = br.open(trackingUrl)
 455 
        response = br.open(trackingUrl)
428 
        page = ungzipResponse(response)
 456 
        page = ungzipResponse(response)
Line 456... Line 484...
456
484
457 
 
 485 
 
458 
 
 486 
 
459 
def main():
 487 
def main():
460 
    store = getStore(1)
 488 
    store = getStore(1)
461 
    br = mechanize.Browser()
 489 
#    br = mechanize.Browser()
462 
    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
 490 
#    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
463 
                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),
 491 
#                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),
464 
                     ('Accept-Encoding', 'gzip,deflate,sdch'),
492 
#                     ('Accept-Encoding', 'gzip,deflate,sdch'),
465 
                     ('Accept-Language', 'en-US,en;q=0.8'),
493 
#                     ('Accept-Language', 'en-US,en;q=0.8'),
466 
                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]
 494 
#                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]
467 
    store.scrapeStoreOrders()
 495 
#    store.scrapeStoreOrders()
468 
    #store.parseOrderRawHtml(2121, "13232", 14, readSSh("/home/amit/917.html"), "https://www.amazon.in/gp/buy/spc/handlers/static-submit-decoupled.html/ref=ox_spc_place_order?ie=UTF8&hasWorkingJavascript=")
 496 
    #store.parseOrderRawHtml(2121, "13232", 14, readSSh("/home/amit/917.html"), "https://www.amazon.in/gp/bOy/spc/handlers/static-submit-decoupled.html/ref=ox_spc_place_order?ie=UTF8&hasWorkingJavascript=")
- 
 
 497 
    #readSSh("/tmp/User157/*")
- 
 
 498 
    store.scrapeAffiliate()
469 
def readSSh(fileName):
 499 
def readSSh(fileName):
470 
    try:
 500 
    try:
471 
            str1 = open(fileName).read()
 501 
            str1 = open(fileName).read()
472 
            return str1
 502 
            return str1
473 
    except:
 503 
    except: