Subversion Repositories SmartDukaan

Rev

Rev 16070 | Rev 16086 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 16070 Rev 16085
Line 9... Line 9...
9 
import optparse
 9 
import optparse
10 
from dtr.storage.MemCache import MemCache
 10 
from dtr.storage.MemCache import MemCache
11 
from dtr.utils.utils import getCashBack
 11 
from dtr.utils.utils import getCashBack
12 
import traceback
 12 
import traceback
13 
from operator import itemgetter
 13 
from operator import itemgetter
- 
 
 14 
import chardet
14 
 
 15 
 
15 
con = None
 16 
con = None
16 
 
 17 
 
17 
parser = optparse.OptionParser()
 18 
parser = optparse.OptionParser()
18 
parser.add_option("-m", "--m", dest="mongoHost",
 19 
parser.add_option("-m", "--m", dest="mongoHost",
Line 117... Line 118...
117 
    isBuyBox = 1
 118 
    isBuyBox = 1
118 
    stock = 0
 119 
    stock = 0
119 
    try:
 120 
    try:
120 
        req = urllib2.Request(url,headers=headers)
 121 
        req = urllib2.Request(url,headers=headers)
121 
        response = urllib2.urlopen(req)
 122 
        response = urllib2.urlopen(req)
- 
 
 123 
        snapdeal_data = response.read()
122 
    except:
 124 
    except:
123 
        print "Unable to scrape %d"%(data['_id'])
 125 
        print "Unable to scrape %d"%(data['_id'])
124 
        return
 126 
        return
- 
 
 127 
    encoding =  chardet.detect(snapdeal_data)
125 
    try:
 128 
    try:
- 
 
 129 
        snapdeal_data = snapdeal_data.decode(encoding.get('encoding'))
- 
 
 130 
    except:
- 
 
 131 
        snapdeal_data = snapdeal_data.decode(encoding.get('latin-1'))
- 
 
 132 
    try:
126 
        vendorInfo = json.load(response)
 133 
        vendorInfo = json.loads(snapdeal_data)
127 
    except:
 134 
    except:
128 
        return
 135 
        return
129 
    finally:
 136 
    finally:
130 
        response.close()
 137 
        response.close()
131 
 
 138