Subversion Repositories SmartDukaan

from BeautifulSoup import BeautifulSoup, NavigableString

from xml.dom.minidom import parseString

import sys

import urllib

import  datetime

import hashlib

import grequests

import hmac

import re

import base64

invalid_tags = ['b', 'i', 'u']

from time import strftime, gmtime

bestSellers = []

from requests import request

def strip_tags(html, invalid_tags):

from operator import itemgetter

            for c in tag.contents:

class MWS(object):

                s += unicode(c)

    URI = "/"

            tag.replaceWith(s)

    VERSION = "2009-01-01"

    return soup

    NS = ''

class AmazonAsyncScraper:

    def __init__(self, access_key, secret_key, merchant_id,

    def __init__(self):

                 domain='https://mws-eu.amazonservices.com', uri="", version=""):

        self.count_trials = 0

        self.access_key = access_key

    def read(self, urls, findStore):

        self.secret_key = secret_key

        returnMap = {}

        self.merchant_id = merchant_id

        print datetime.datetime.now()

        self.domain = domain

                  'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',

        self.uri = uri or self.URI

                  'Accept-Encoding':'gzip,deflate,sdch'

        self.version = version or self.VERSION

        rs = (grequests.get(u, headers=header) for u in urls)

    def make_request(self, extra_data, action,method="GET", **kwargs):

        for x in grequests.map(rs):

        params = {

            soup = strip_tags(x.text,invalid_tags)

            'AWSAccessKeyId': self.access_key,

            for tag in soup.findAll(True):

            'SignatureVersion': '2',

                if tag.name in invalid_tags:

            'Timestamp': self.get_timestamp(),

                    s = ""

            'Version': self.version,

                    for c in tag.contents:

            'SignatureMethod': 'HmacSHA256',

                        s += unicode(c)

            'Action': action

        }

                    tag.replaceWith(s)

        if action=='GetLowestOfferListingsForSKU':

            dataLength = len(sellerData)

            params['ExcludeMe']='true'

            for data in sellerData:

        params.update(extra_data)

                price = data.find('span', attrs={'class' : re.compile('.*olpOfferPrice*')}).find('span').text

        request_description = '&'.join(['%s=%s' % (k, urllib.quote(params[k], safe='-_.~').encode('utf-8')) for k in sorted(params)])

                unitCost = float(price.replace("Rs.","").replace(",",""))

        signature = self.calc_signature(method, request_description)

                shippingCost = data.find('p', attrs={'class' : re.compile('.*olpShippingInfo*')}).find('span').text

        url = '%s%s?%s&Signature=%s' % (self.domain, self.uri, request_description, urllib.quote(signature))

                sellerColumn =  data.find('p', attrs={'class' : re.compile('.*olpSellerName*')})

        headers = {'User-Agent': 'AmazonJavascriptScratchpad/1.0 (Language=Python)'}

                    temp =  sellerColumn.find('a')

        headers.update(kwargs.get('extra_headers', {}))

                    if len(store)!=0:

        response = request(method, url)

                    tempMap['storeUrl'] =storeUrl

        if action=='GetLowestOfferListingsForSKU':

                    tempMap['sellerPrice'] = unitCost+shippingCost

            return self.parse_competitor_pricing_response(response)

                info.append(tempMap) 

        elif action=='GetMyPriceForSKU':

            return self.findStoreFront(returnMap)

            return self.parse_my_pricing_response(response)

            return returnMap

            raise

    def findStoreFront(self,returnMap):

    def parse_competitor_pricing_response(self,response):

        storeFront={}

        spString = "<dom>" + spString + "</dom>"

        for arr in returnMap.itervalues():

        dom = parseString(spString)

            for dic in arr:

        offerMap = {}

                if dic['isStoreFront']!='True':

        for skuOffer in skuOffers:

                    storeFront[dic.get('storeUrl')] =''

            status = skuOffer.attributes.items()[0][1]

        rs = (grequests.get(u,stream=False) for u in storeFront.keys())

            sku = skuOffer.attributes.items()[1][1]

        for x in grequests.map(rs):

            info = []

            soup = strip_tags(x.text,invalid_tags)

            for offer in skuOffer.getElementsByTagName('LowestOfferListing'):

            x.close

                if len(info)==3:

            #print x.url.rfind('&me=')

                    break

            #print x.url[x.url.rfind('&me='):].rfind('&')

                temp = {}

            mId= x.url[x.url.rfind('&me=')+4:x.url[x.url.rfind('&me='):].rfind('&')+x.url.rfind('&me=')]

                amount = offer.getElementsByTagName('Amount')[0].firstChild.nodeValue

            sellerName = soup.title.string

                temp['sellingPrice'] = amount

            #print mId

                temp['promoPrice'] = amount

            try:

                try:

                sellerName = sellerName[0:ind].strip()

                    temp['shippingTime'] = (offer.getElementsByTagName('Max')[0].firstChild.nodeValue).replace('days','')

            except:

                except:

                    ind = sellerName.split(":")

                        a =  str_rating.split()

                    sellerName = ind[1].strip()

                            if x.isdigit():

                    sellerName =""

                    temp['rating'] = '0'

            #storeFront[re.compile('*'+mId+'.*')] = sellerName

                temp['notOurSku'] = True

            #print mId

                info.append(temp)

            #print sellerName

        return offerMap

            myRe = re.compile('.*'+mId+'.*')

        spString = re.sub('<\?.*\?>','',response.text)

            for key in storeFront:

        dom = parseString(spString)

                if myRe.match(key):

        for skuOffer in skuOffers:

                    #print "Match found ",key

            sku = skuOffer.attributes.items()[1][1]

                    storeFront[key] = sellerName.strip()

                temp['sellingPrice'] = regularPrice

            #storeFront.get(re.compile('.*'+mId+'.*'))

                temp['promoPrice'] = promoPrice

        for arr in returnMap.itervalues():

                if promoPrice == regularPrice:

            #print "arr is ",arr

                    temp['promotion'] = False

            for dic in arr:

                else:

                #print "dic ",dic

                temp['asin'] = asin

                if dic['isStoreFront']!='True':

                temp['notOurSku'] = False

                    dic['sellerName'] =storeFront.get(dic.get('storeUrl'))

                temp['fulfillmentChannel'] ='AMAZON'

                    dic['isStoreFront']='True'

                temp['rating'] = '0'