Subversion Repositories SmartDukaan

#!/usr/bin/python

"""

 This script prints date of run, search-string, and position and page number where saholic result 
 first appears in google search results.

"""

import sys, pycurl, re
import datetime, time
import random
import MySQLdb



# some initial setup:
USER_AGENT = 'Mozilla/4.0 (compatible; MSIE 6.0)'
# USER_AGENT = 'Mozilla/5.0'
FIND_DOMAIN = 'www.saholic.com'
LOCALE = '.co.in'
MAX_PAGE = 10
NUM_PER_PAGE = 100
SEARCH_STRING_MAX_SLEEP = 120 #seconds
PAGE_MAX_SLEEP = 20 #seconds
DB_HOST = "localhost"
DB_USER = "root"
DB_PASSWORD = "shop2020"
DB_NAME = "serp"

# define class to store result:
class ResposeStorage:
  def __init__(self):
    self.contents = ''

  def body_callback(self, buf):
    self.contents = self.contents + buf

class RankCheck:
  def __init__(self, searchStringMaxSleep=SEARCH_STRING_MAX_SLEEP, pageSleep=PAGE_MAX_SLEEP):
    self.searchStringMaxSleep = searchStringMaxSleep
    self.pageSleep = pageSleep
    self.userAgent = USER_AGENT
    self.findDomain = FIND_DOMAIN
    self.locale = LOCALE
    self.maxPage = MAX_PAGE
    self.numPerPage = NUM_PER_PAGE
    self.searchStrings = []
    self.results = []
    self.db_mode = False
    self.breakAfterNextItr = False

  def addSearchStrings(self, searchStrings):
    self.searchStrings.extend(searchStrings)  

  def init_curl(self):
    # set up curl:
    rankRequest = pycurl.Curl()
    rankRequest.setopt(pycurl.USERAGENT, USER_AGENT)
    rankRequest.setopt(pycurl.FOLLOWLOCATION, 1)
    rankRequest.setopt(pycurl.AUTOREFERER, 1)
    rankRequest.setopt(pycurl.COOKIEFILE, '')
    rankRequest.setopt(pycurl.HTTPGET, 1)
    rankRequest.setopt(pycurl.REFERER, '')
    return rankRequest

  def start(self):
    for search_string in self.searchStrings:
      if len(self.results) >= 30:
        time.sleep(random.randint(60*5, 60*15)) # sleep for 5 to 15 min after 20 queries
        if self.db_mode:
          self.pushResultsToDb()
      self.find_google_position(search_string)
      if self.db_mode:
        time.sleep(random.randint(0, self.searchStringMaxSleep))
        
      if self.breakAfterNextItr:
        break
    
    if(len(self.results) > 0 and self.db_mode):
      self.pushResultsToDb()
  
  def search_page(self, page, page_url):
    # instantiate curl and result objects:
    rankCheck = ResposeStorage();
    rankRequest = self.init_curl()
    rankRequest.setopt(pycurl.WRITEFUNCTION, rankCheck.body_callback)
    rankRequest.setopt(pycurl.URL, page_url + '&start=' + str(page * NUM_PER_PAGE))
    rankRequest.perform()
    # close curl:
    rankRequest.close()
    
    # collect the search results
    html = rankCheck.contents
    counter = page * NUM_PER_PAGE
    result = 0
    this_url = ""
    
    if html.count("Our systems have detected unusual traffic from your computer network.") > 0:
      print "Blocked by Google"
      self.breakAfterNextItr = True
      return -1, ""
    url = unicode(r'(<h3 class="r"><a href=")((https?):((//))+[\w\d:#@%/;$()~_?\+-=\\\.&]*)')
    for google_result in re.finditer(url, html):
      # print m.group()
      this_url = google_result.group()
      this_url = this_url[23:]
      counter += 1
      
      google_url_regex = re.compile("((https?):((//))+([\w\d:#@%/;$()~_?\+-=\\\.&])*" + FIND_DOMAIN + "+([\w\d:#@%/;$()~_?\+-=\\\.&])*)")
      google_url_regex_result = google_url_regex.match(this_url)
      if google_url_regex_result:
        result = counter
        break
      
    return result, this_url

  def find_google_position(self, search_string):
    ENGINE_URL = 'http://www.google' + LOCALE + '/search?q=' + search_string.replace(' ', '+') + '&num=' + str(NUM_PER_PAGE)
    # print ENGINE_URL
    
    # run curl:
    for i in range(0, MAX_PAGE):
      (position, url) = self.search_page(i, ENGINE_URL)
      time.sleep(random.randint(0, self.pageSleep))
      
      if position != 0:
        break
  
    if position ==-1:
      return
    if position == 0:
      position = NUM_PER_PAGE * MAX_PAGE
      url = ""
      
    self.results.append((position, ((position-1)/10 + 1), url, search_string))
    print "{0:s}, {1:s}, {2:d}, {3:d}, {4:s}".format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M"), search_string, position, (position-1)/10 + 1, url)
    
  def getDbConnection(self):
    return MySQLdb.connect(DB_HOST, DB_USER, DB_PASSWORD, DB_NAME)
  
  def closeConnection(self, conn):
    conn.close()

  def loadSearchStringsFromDb(self):
    conn = self.getDbConnection()
    
    # Prepare SQL query to INSERT a record into the database.
    # sql = "SELECT query FROM query where is_active = 1"
    sql = "SELECT query FROM query where is_active = 1 and id%2 = dayofweek(now())%2"
    try:
      # prepare a cursor object using cursor() method
      cursor = conn.cursor()
      # Execute the SQL command
      cursor.execute(sql)
      # Fetch all the rows in a list of lists.
      results = cursor.fetchall()
      for row in results:
        self.searchStrings.append(row[0])
      cursor.close()
    except Exception as e:
      print "Error: unable to fetch data"
      print e
      
    self.closeConnection(conn)

  def pushResultsToDb(self):
    conn = self.getDbConnection()
    
    try:
      # prepare a cursor object using cursor() method
      cursor = conn.cursor()
       
      cursor.executemany (
          """
              insert into rank(query_id, date, position, page, url) 
              select id , now(), %s, %s, %s from query where query = %s;
          """, self.results)
      conn.commit()
      self.results = []
      cursor.close()
    except Exception as e:
      print "Error: unable to insert row"
      print e
    
    self.closeConnection(conn)


  
def main():
  if len(sys.argv) > 1:
    rank_cheker = RankCheck(0, 0)
    rank_cheker.addSearchStrings(sys.argv[1:])
  else:
    rank_cheker = RankCheck()
    rank_cheker.loadSearchStringsFromDb()
    rank_cheker.db_mode = True

  rank_cheker.start()

if __name__ == '__main__':
    main()