Subversion Repositories SmartDukaan

from BeautifulSoup import BeautifulSoup

import mechanize

import urllib2

import sys

import urllib

import cookielib

import urllib

def getBrowserObject():

    br = mechanize.Browser(factory=mechanize.RobustFactory())

    cj = cookielib.LWPCookieJar()

    br.set_cookiejar(cj)

    br.set_handle_equiv(True)

    br.set_handle_redirect(True)

    br.set_handle_referer(True)

    br.set_handle_robots(False)

    br.set_debug_http(False)

    br.set_debug_redirects(False)

    br.set_debug_responses(False)

    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)

    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),

                     ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),

                     ('Accept-Encoding', 'gzip,deflate,sdch'),                  

                     ('Accept-Language', 'en-US,en;q=0.8'),                     

                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]

    return br

def login(url):

    br = getBrowserObject()

    br.open(url)

    response = br.open(url)

    ungzipResponse(response, br)

    html = response.read()

    br.select_form(name="login")

    br.form['unmae'] = "saholic"

    br.form['pword'] = "2020shop"

    response = br.submit()

    print "********************"

    print "Attempting to Login"

    print "********************"

    ungzipResponse(response, br)

    return br

def fetchItemDetails(merchant_url,br):

    response = br.open(merchant_url)

    ungzipResponse(response, br)

    page = response.read()

    page=page.decode("utf-8")

    soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

    table = soup.find("table" , {"class" : "stdtable tablesorter"})

    table_rows = soup.findAll("tr" , {"class" : "gradeX"})

    data = []

    for tr in table_rows:

        row_count=1

        new = []

        for t in tr:

            if row_count==4:

                print "********************"

                print "Product name field :",

                te =str(t)

                start = te.rindex("mid=\"") + len( "mid=\"" )

                end = te.rindex( "\">", start )

                parse= te[start:end]

                print parse

                new.append(parse)

            if row_count==8:

                print "********************"

                print "Cheapest price :",t.text

                cheapest_price = t.text

                cheapest_price = cheapest_price.replace("Rs.",'')

                cheapest_price =  int(cheapest_price.replace(",",''))

                new.append(cheapest_price)

            if row_count==16:

                print "********************"

                print "Saholic price :",t.text

                our_price = t.text

                our_price = our_price.replace("Rs.",'')

                our_price =  int(our_price.replace(",",''))

                new.append(our_price)

            row_count+=1

        data.append(new)

    print "******************************"

    print "Data Populated from 91 Mobiles"

    print "******************************"

    return data

def getSaholicEntityId(data):

    br = getBrowserObject()

    for i in data:

        response = br.open("http://www.91mobiles.com/redir.php?origin=detail&mobileid=%s&storename=saholic.com&sf=&storeflag=1&cc="%(i[0]))

        ungzipResponse(response, br)

        page = response.read()

        page=page.decode("utf-8")

        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)

        link = soup.find("body" , {"class" : ""})

        for a in soup.findAll('a', href=True):

            url = str(a['href'])

            end = url.rindex("?afid")

            our_url = url[0:end]

            length= len(our_url)

            ind = our_url.rfind("-")

            entityId = url[ind+1:length]

            print "*****************************"

            print "Trying to fetch entity id...."

            print "EntityId :",entityId   

            print "*****************************"

def ungzipResponse(r,b):

    headers = r.info()

    if headers['Content-Encoding']=='gzip':

        import gzip

        print "********************"

        print "Deflating gzip response"

        print "********************"

        gz = gzip.GzipFile(fileobj=r, mode='rb')

        html = gz.read()

        gz.close()

        headers["Content-type"] = "text/html; charset=utf-8"

        r.set_data( html )

        b.set_response(r)

def main():

    print "Opening 91 Mobiles merchant login page"

    login_url = "http://www.91mobiles.com/91merchants/login.php"

    merchant_url = "http://www.91mobiles.com/91merchants/manage_merchants.php"

    br = login(login_url)

    itemData = fetchItemDetails(merchant_url,br)

    getSaholicEntityId(itemData)

if __name__ == "__main__":

    main()