Subversion Repositories SmartDukaan

Rev

Rev 14708 | Rev 14737 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14708 Rev 14736
Line 1... Line 1...
1 
import time
 1 
import time
2 
import pymongo
 2 
import pymongo
- 
 
 3 
import urllib2
- 
 
 4 
import gzip
- 
 
 5 
import StringIO
3 
 
 6 
 
4 
#TODO Need to add messy stuff to conf.
 7 
#TODO Need to add messy stuff to conf.
5 
con=None
 8 
con=None
- 
 
 9 
headers = {
- 
 
 10 
            'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',
- 
 
 11 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
- 
 
 12 
            'Accept-Language' : 'en-US,en;q=0.8',
- 
 
 13 
            'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
- 
 
 14 
            'Connection':'keep-alive',
- 
 
 15 
            'Accept-Encoding' : 'gzip,deflate,sdch'
- 
 
 16 
        }
- 
 
 17 
 
6 
def get_mongo_connection(host='localhost', port=27017):
 18 
def get_mongo_connection(host='localhost', port=27017):
7 
    global con
 19 
    global con
8 
    if con is None:
 20 
    if con is None:
9 
        print "Establishing connection %s host and port %d" %(host,port)
 21 
        print "Establishing connection %s host and port %d" %(host,port)
10 
        try:
 22 
        try:
Line 63... Line 75...
63 
        if not itemCashBackMap.has_key(row['skuId']):
 75 
        if not itemCashBackMap.has_key(row['skuId']):
64 
            itemCashBackMap[row['skuId']] = row
 76 
            itemCashBackMap[row['skuId']] = row
65 
    mc.set("item_cash_back", itemCashBackMap, 24 * 60 * 60)
 77 
    mc.set("item_cash_back", itemCashBackMap, 24 * 60 * 60)
66 
    mc.set("category_cash_back", cashBackMap, 24 * 60 * 60)
 78 
    mc.set("category_cash_back", cashBackMap, 24 * 60 * 60)
67 
 
 79 
 
- 
 
 80 
def ungzipResponse(r):
- 
 
 81 
    headers = r.info()
- 
 
 82 
    if headers.get('Content-Encoding')=='gzip':
- 
 
 83 
        print "********************"
- 
 
 84 
        print "Deflating gzip response"
- 
 
 85 
        print "********************"
- 
 
 86 
        url_f = StringIO.StringIO(r.read())
- 
 
 87 
        gz = gzip.GzipFile(fileobj=url_f)
- 
 
 88 
        html = gz.read()
- 
 
 89 
        gz.close()
- 
 
 90 
        return html
- 
 
 91 
    return r.read()
- 
 
 92 
 
- 
 
 93 
 
- 
 
 94 
def fetchResponseUsingProxy(url, headers=headers):
- 
 
 95 
    proxy = urllib2.ProxyHandler({'http': 'http://192.161.163.60:8800'})
- 
 
 96 
    opener = urllib2.build_opener(proxy)
- 
 
 97 
    urllib2.install_opener(opener)
- 
 
 98 
    req = urllib2.Request(url,headers=headers)
- 
 
 99 
    response = urllib2.urlopen(req)
- 
 
 100 
    response_data = ungzipResponse(response)
- 
 
 101 
    response.close()
- 
 
 102 
    return response_data
- 
 
 103
- 
 
 104
- 
 
 105 
 
- 
 
 106