Subversion Repositories SmartDukaan

import urllib2
from BeautifulSoup import BeautifulSoup, NavigableString
import re
from dtr.utils.utils import fetchResponseUsingProxy

invalid_tags = ['b', 'i', 'u']
bestSellers = []

def strip_tags(html, invalid_tags):
    soup = BeautifulSoup(html,convertEntities=BeautifulSoup.HTML_ENTITIES)

    for tag in soup.findAll(True):
        if tag.name in invalid_tags:
            s = ""

            for c in tag.contents:
                if not isinstance(c, NavigableString):
                    c = strip_tags(unicode(c), invalid_tags)
                s += unicode(c)

            tag.replaceWith(s)

    return soup

class AmazonScraper:
    def __init__(self,livePricing=None):
        self.count_trials = 0
        self.livePricing = livePricing
    
    def read(self, url):
        response_data = ""
        try:
            response_data = fetchResponseUsingProxy(url, livePricing=self.livePricing)
        except Exception as e:
            print 'ERROR: ', e
            print 'Retrying'
            self.count_trials += 1
            
            if self.count_trials < 5:
                return self.read(url)
        
        self.response_data=response_data
        if "Server Busy" in self.response_data:
            print "Server busy...Ahhhhh"
            self.count_trials += 1
            return self.read(url)
        return self.createData()
    
    def createData(self):
        self.soup = strip_tags(self.response_data,invalid_tags)
        self.response_data =None
        return self.scrape(self.soup)
    
    
    def scrape(self,soup):
        try:
            sellerData = soup.find("span" , {"id" : "priceblock_dealprice"})
            dealPrice = float(sellerData.text.replace("Rs.","").replace(",",""))
            print dealPrice
        except:
            dealPrice = 0.0
        try:
            dealStatus = soup.find('span',{'id':re.compile('dealStatusAvailability_*')})
            dealStatus = float(dealStatus.text.replace("%","").replace(",",""))
        except:
            dealStatus = 100
        
        
        if dealStatus < 100 and dealPrice > 0:
            return dealPrice
        else:
            return 0.0

if __name__ == '__main__':
    scraper = AmazonScraper(True)
    print scraper.read('http://www.amazon.in/dp/B015CSIA38/ref=gbdp_vlo_61287013_B015CSIA38?_encoding=UTF8&smid=A2WBY8FP973J47')