Subversion Repositories SmartDukaan

Rev

Rev 18284 | Rev 19645 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 18284 Rev 19240
Line 44... Line 44...
44 
 
 44 
 
45 
def getSoupObject(url):
 45 
def getSoupObject(url):
46 
    print "Getting soup object for"
 46 
    print "Getting soup object for"
47 
    print url
 47 
    print url
48 
    soup = None
 48 
    soup = None
49 
    response_data = fetchResponseUsingProxy(url)
 49 
    response_data = fetchResponseUsingProxy(url,proxy=False)
50 
    print response_data
 50 
    print response_data
51 
    try:
 51 
    try:
52 
        page=response_data.decode("utf-8")
 52 
        page=response_data.decode("utf-8")
53 
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
 53 
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
54 
    except:
 54 
    except:
Line 57... Line 57...
57 
 
 57 
 
58 
def scrapeBestSellerMobiles():
 58 
def scrapeBestSellerMobiles():
59 
    global bestSellers
 59 
    global bestSellers
60 
    rank = 0
 60 
    rank = 0
61 
    for i in [1, 21, 41, 61, 81]:
 61 
    for i in [1, 21, 41, 61, 81]:
62 
        url = "http://www.flipkart.com/mobiles/~bestsellers/pr?p[]=sort=popularity&sid=tyy,4io&start=%d&ajax=true" %(i)
 62 
        url = "http://www.flipkart.com/lc/pr/pv1/spotList1/spot1/productList?sid=tyy,4io&filterNone=true&start=%d&ajax=true" %(i)
- 
 
 63
- 
 
 64
63 
        soup = getSoupObject(url)
 65 
        soup = getSoupObject(url)
64 
        product_divs = soup.findAll('div',{'class':re.compile('.*browse-product')})
 66 
        product_divs = soup.findAll('div',{'class':re.compile('.*browse-product')})
65 
        for x in product_divs:
 67 
        for x in product_divs:
66 
            rank  = rank +1
 68 
            rank  = rank +1
67 
            print rank,
 69 
            print rank,
Line 73... Line 75...
73 
def scrapeBestSellerTablets():
 75 
def scrapeBestSellerTablets():
74 
    global bestSellers
 76 
    global bestSellers
75 
    bestSellers = []
 77 
    bestSellers = []
76 
    rank = 0
 78 
    rank = 0
77 
    for i in [1, 21, 41, 61, 81]:
 79 
    for i in [1, 21, 41, 61, 81]:
78 
        url = "http://www.flipkart.com/tablets/~bestsellers/pr?p[]=sort=popularity&sid=tyy,hry&start=%d&ajax=true" %(i)
 80 
        url = "http://www.flipkart.com/lc/pr/pv1/spotList1/spot1/productList?sid=tyy,hry&filterNone=true&start=%d&ajax=true" %(i)
79 
        soup = getSoupObject(url)
 81 
        soup = getSoupObject(url)
80 
        product_divs = soup.findAll('div',{'class':re.compile('.*browse-product')})
 82 
        product_divs = soup.findAll('div',{'class':re.compile('.*browse-product')})
81 
        for x in product_divs:
 83 
        for x in product_divs:
82 
            rank  = rank +1
 84 
            rank  = rank +1
83 
            print rank,
 85 
            print rank,