Subversion Repositories SmartDukaan

from BeautifulSoup import BeautifulSoup
import urllib2
import urllib
import MySQLdb
import re
import sys
listCircle = [
['Andhra Pradesh','hyd',1,0],
['Assam','asm',2,0],
['Bihar','bih',3,0],
['Chennai','che',4,0],
['Delhi NCR','del',5,0],
['Gujarat','guj',6,0],
['Haryana','har',9,0],
['Himachal Pradesh','hip',7,0],
['Jammu','jnk',8,0],
['Karnataka','kar',10,0],
['Kerela','ker',11,0],
['Kolkata','kol',12,0],
['MP','map',14,0],
['Maharashtra','mah',13,1],
['Mumbai','mum',15,1],
['North East','ner',16,0],
['Orissa','ors',17,0],
['Punjab','pun',18,0],
['Rajasthan','raj',19,1],
['Tamil Nadu','tam',20,0],
['UP East','upe',21,1],
['UP(West)','upw',22,0],
['West Bengal','rob',23,0]
]

j =0 

db = MySQLdb.connect("localhost","root","shop2020","transaction" )
for i in listCircle:
    print "Circle Name: "+listCircle[j][0]
    pageno=1
    start=1
    while(True):
        url = "https://shop.vodafone.in/shop/rechargeOnline.jsp?start=%d"%(start)
        data = {'cid' : listCircle[j][1]}
        Attempt = 1
        connected = False
        while not connected:
            try:
                data = urllib.urlencode(data)
                req = urllib2.Request(url, data)
                response = urllib2.urlopen(req)
                page = response.read()
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
                connected =True
            except:
                print "In Except Block for Url "+url+" And Circle "+listCircle[j][0]
                print "Retrying:",
                print Attempt
                Attempt = Attempt+1
                if Attempt ==6:
                    sys.exit(1)
        links = soup.findAll("a", {"class" : "arrowlink"})            
        size = len(links)
        if size == 0:
            print "Break for url "+url
            break
        print "Page: ",
        print pageno 
        for i in links:
            connected_plans = False
            Attempt_Plans =1
            result = re.search('sku[0-9]+', str(i))
            sku = result.group(0)
            print "SkuId:",
            print sku
            table_url = "https://shop.vodafone.in/shop/ajax/viewRechargeDetails.jsp?id="+sku
            while not connected_plans:
                try:    
                    req = urllib2.Request(table_url)
                    response = urllib2.urlopen(req)
                    page = response.read()
                    soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
                    connected_plans = True
                except:
                    print "Except Block Hitting SkuUrl "
                    print "Retrying:",
                    print Attempt_Plans
                    Attempt_Plans = Attempt_Plans+1
                    if Attempt_Plans ==6:
                        sys.exit(2)
            Table = soup.find("table")
            td = Table.findAll("td")
            column_count = 1
            for value in td:
                if column_count == 2:
                    Denomination = value.text
                    print "Deno: "+Denomination
                    Denomination = float(Denomination)
                    column_count = column_count+1
                    if Denomination < 10:
                        break
                elif column_count ==7:    
                    Talktime = value.text
                    print "Talktime String: "+Talktime
                    Talktime = float(Talktime)
                    print "Talktime: ",
                    print Talktime
                    column_count = column_count+1
                elif column_count ==4:
                    RechargeType = value.text
                    column_count = column_count+1
                    print "RechargeType: "+RechargeType
                elif column_count ==11:
                    Benefit = value.text
                    column_count = column_count+1
                elif column_count ==13:
                    Validity = value.text
                    column_count = column_count+1
                    print "Validity: "+Validity
                elif column_count ==15:
                    Deduction = value.text
                    print 'Deduction: '+Deduction
                    Deduction = float(Deduction)
                    column_count = column_count+1
                    if Deduction > (Denomination/2):
                        IsSpecial = 2
                    else:
                        IsSpecial = 1
                    if Validity == '-' or Validity == 'NA' or Validity =='na' or Validity =='nil' or Validity =='' or Validity =='Nil' or Validity =='NIL':
                        Validity = '0 days'
                    if IsSpecial ==1 and (Validity =='lifetime' or Validity =='Lifetime' or Validity =='life time' or Validity=='Life Time' or Validity =='Life time' or Validity =='Lifetime Validity' or Validity== 'UNLIMITED' or Validity== 'unlimited' or Validity== 'Unlimited'):
                        Validity='0 days'
                    if 'Full Talktime' in Benefit:
                        print "****-----Full Talktime----****"
                    if  not('Full Talktime' in Benefit) and listCircle[j][3] == 1 and Denomination > 1000 and IsSpecial ==1:
                            print "Skipping Bogus Plans"
                            break
                    if  not('Full Talktime' in Benefit) and listCircle[j][3] == 1 and Denomination%100 !=0 and Denomination > 50 and IsSpecial ==1:
                            print "Skipping Bogus Plans"
                            break
                    if IsSpecial ==1:
                        Benefit = str(Talktime)
                        Benefit = "Talktime of Rs "+Benefit
                    else:
                        print "Benefit: "+Benefit
                    cursor = db.cursor()
                    sql = '''INSERT INTO rechargedenomination (operatorId, circleId, denominationType, validity, amount, description)\
                    VALUES ('%d', '%d', '%d', '%s', '%s', "%s")''' % \
                    (25, listCircle[j][2], IsSpecial, Validity, Denomination, Benefit)
                    cursor.execute(sql)
                    db.commit()
                else:
                    column_count = column_count+1
        start = start+10
        pageno = pageno+1
    j = j+1  
db.close()