Subversion Repositories SmartDukaan

Rev

Rev 17040 | Rev 17068 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 17040 Rev 17067
Line 1... Line 1...
1 
import urllib2
 1 
import urllib2
2 
from BeautifulSoup import BeautifulSoup
 2 
from BeautifulSoup import BeautifulSoup
3 
import pymongo
 - 
 
4 
import re
 3 
import re
5 
from dtr.utils.utils import to_java_date
 4 
from dtr.utils.utils import to_java_date
6 
import optparse
 5 
import optparse
7 
from datetime import datetime
 6 
from datetime import datetime
8 
import smtplib
 7 
import smtplib
9 
from email.mime.text import MIMEText
 8 
from email.mime.text import MIMEText
10 
from email.mime.multipart import MIMEMultipart
 9 
from email.mime.multipart import MIMEMultipart
11 
from dtr.utils.utils import fetchResponseUsingProxy, get_mongo_connection, ungzipResponse
 10 
from dtr.utils.utils import fetchResponseUsingProxy, get_mongo_connection, ungzipResponse
12 
from pyquery import PyQuery
 - 
 
13 
import json
 11 
import json
14 
import mechanize
 - 
 
15 
import urllib
 12 
import urllib
16 
 
 13 
 
17 
 
 14 
 
18 
con = None
 15 
con = None
19 
parser = optparse.OptionParser()
 16 
parser = optparse.OptionParser()
Line 49... Line 46...
49 
        self.category = category
 46 
        self.category = category
50 
        self.thumbnail = thumbnail
 47 
        self.thumbnail = thumbnail
51 
        self.source_product_name = source_product_name
 48 
        self.source_product_name = source_product_name
52 
        self.marketPlaceUrl = marketPlaceUrl
49 
        self.marketPlaceUrl = marketPlaceUrl
53
50
54 
def getBrowserObject():
 - 
 
55 
    import cookielib
 - 
 
56 
    br = mechanize.Browser(factory=mechanize.RobustFactory())
 - 
 
57 
    cj = cookielib.LWPCookieJar()
 - 
 
58 
    br.set_cookiejar(cj)
 - 
 
59 
    br.set_handle_equiv(True)
 - 
 
60 
    br.set_handle_redirect(True)
 - 
 
61 
    br.set_handle_referer(True)
 - 
 
62 
    br.set_handle_robots(False)
 - 
 
63 
    br.set_debug_http(False)
 - 
 
64 
    br.set_debug_redirects(False)
 - 
 
65 
    br.set_debug_responses(False)
 - 
 
66
- 
 
67 
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
 - 
 
68
- 
 
69 
    br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
 - 
 
70 
                     ('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),
 - 
 
71 
                     ('Accept-Encoding', 'gzip,deflate,sdch'),
- 
 
72 
                     ('Accept-Language', 'en-US,en;q=0.8'),
- 
 
73 
                     ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'),
 - 
 
74 
                     ('Host' , 'm.homeshop18.com'),
 - 
 
75 
                     ('Referer', 'http://m.homeshop18.com/search.mobi?keyword=&categoryId=3027&sortConfig=BEST_SELLER')
 - 
 
76 
                     ]
 - 
 
77 
    return br
 - 
 
78 
 
 - 
 
79 
def ungzipResponseBr(r,b):
 - 
 
80 
    headers = r.info()
 - 
 
81 
    if headers['Content-Encoding']=='gzip':
 - 
 
82 
        import gzip
 - 
 
83 
        print "********************"
 - 
 
84 
        print "Deflating gzip response"
 - 
 
85 
        print "********************"
 - 
 
86 
        gz = gzip.GzipFile(fileobj=r, mode='rb')
 - 
 
87 
        html = gz.read()
 - 
 
88 
        gz.close()
 - 
 
89 
        headers["Content-type"] = "text/html; charset=utf-8"
 - 
 
90 
        r.set_data( html )
 - 
 
91 
        b.set_response(r)
 - 
 
92
- 
 
93 
def getCsrfValue():
 - 
 
94 
    global csrfValue
 - 
 
95 
    csrfValUrl = 'http://m.homeshop18.com/search.mobi?keyword=&categoryId=14569&sortConfig=BEST_SELLER'
 - 
 
96 
    data = fetchResponseUsingProxy(csrfValUrl, proxy=False )
 - 
 
97 
    pq = PyQuery(data)
 - 
 
98 
    csrf = pq("input[name=csrf]")
 - 
 
99 
    csrfValue = csrf[0].value
 - 
 
100 
    print csrfValue
 - 
 
101
- 
 
102 
def commitBestSellers(category):
 51 
def commitBestSellers(category):
103 
    global exceptionList
 52 
    global exceptionList
104 
    print "Rank",
 53 
    print "Rank",
105 
    print '\t',
 54 
    print '\t',
106 
    print 'Identifier'
 55 
    print 'Identifier'