| Line 4... |
Line 4... |
| 4 |
import pymongo
|
4 |
import pymongo
|
| 5 |
import time
|
5 |
import time
|
| 6 |
import urllib
|
6 |
import urllib
|
| 7 |
import urllib2
|
7 |
import urllib2
|
| 8 |
from datetime import datetime
|
8 |
from datetime import datetime
|
| - |
|
9 |
import random
|
| 9 |
#TODO Need to add messy stuff to conf.
|
10 |
#TODO Need to add messy stuff to conf.
|
| 10 |
con=None
|
11 |
con=None
|
| 11 |
headers = {
|
12 |
headers = {
|
| 12 |
'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',
|
13 |
'User-agent':'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36',
|
| 13 |
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
14 |
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
| 14 |
'Accept-Language' : 'en-US,en;q=0.8',
|
15 |
'Accept-Language' : 'en-US,en;q=0.8',
|
| 15 |
'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
|
16 |
'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
|
| 16 |
'Connection':'keep-alive',
|
17 |
'Connection':'keep-alive',
|
| 17 |
'Accept-Encoding' : 'gzip,deflate,sdch'
|
18 |
'Accept-Encoding' : 'gzip,deflate,sdch'
|
| 18 |
}
|
19 |
}
|
| 19 |
PROXY_URL = "http://192.161.160.97:8800/"
|
20 |
PROXY_URL = "http://192.161.160.203:8800/"
|
| - |
|
21 |
PROXY_LIST = ['192.161.160.203:8800','173.234.194.214:8800', \
|
| - |
|
22 |
'192.161.160.13:8800','173.208.39.246:8800','192.161.160.212:8800', \
|
| - |
|
23 |
'173.234.194.37:8800','173.208.39.67:8800','192.161.163.249:8800', \
|
| - |
|
24 |
'192.161.163.60:8800']
|
| - |
|
25 |
|
| 20 |
PUSH_NOTIFICATION_URL='http://api.profittill.com/admin/users/push'
|
26 |
PUSH_NOTIFICATION_URL='http://api.profittill.com/admin/users/push'
|
| 21 |
DTR_API_BASIC_AUTH = base64.encodestring('%s:%s' % ("dtr", "dtr18Feb2015")).replace('\n', '')
|
27 |
DTR_API_BASIC_AUTH = base64.encodestring('%s:%s' % ("dtr", "dtr18Feb2015")).replace('\n', '')
|
| 22 |
|
28 |
|
| 23 |
def get_mongo_connection(host='localhost', port=27017):
|
29 |
def get_mongo_connection(host='localhost', port=27017):
|
| 24 |
global con
|
30 |
global con
|
| Line 92... |
Line 98... |
| 92 |
return html
|
98 |
return html
|
| 93 |
return r.read()
|
99 |
return r.read()
|
| 94 |
|
100 |
|
| 95 |
|
101 |
|
| 96 |
def fetchResponseUsingProxy(url, headers=headers):
|
102 |
def fetchResponseUsingProxy(url, headers=headers):
|
| - |
|
103 |
PROXY_URL = PROXY_LIST[random.randint(0, len(PROXY_LIST) -1)]
|
| - |
|
104 |
print PROXY_URL
|
| 97 |
proxy = urllib2.ProxyHandler({'http': PROXY_URL})
|
105 |
proxy = urllib2.ProxyHandler({'http': PROXY_URL})
|
| 98 |
opener = urllib2.build_opener(proxy)
|
106 |
opener = urllib2.build_opener(proxy)
|
| 99 |
urllib2.install_opener(opener)
|
107 |
urllib2.install_opener(opener)
|
| 100 |
req = urllib2.Request(url,headers=headers)
|
108 |
req = urllib2.Request(url,headers=headers)
|
| 101 |
response = urllib2.urlopen(req)
|
109 |
response = urllib2.urlopen(req)
|