Subversion Repositories SmartDukaan

Rev

Rev 14380 | Rev 19240 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14380 Rev 18284
Line 1... Line 1...
1 
import urllib2
 1 
import urllib2
2 
from BeautifulSoup import BeautifulSoup
 2 
from BeautifulSoup import BeautifulSoup
3 
import pymongo
 3 
import pymongo
4 
import re
 4 
import re
5 
from dtr.utils.utils import to_java_date
 5 
from dtr.utils.utils import to_java_date, fetchResponseUsingProxy
6 
from datetime import datetime
 6 
from datetime import datetime
7 
import optparse
 7 
import optparse
8 
import smtplib
 8 
import smtplib
9 
from email.mime.text import MIMEText
 9 
from email.mime.text import MIMEText
10 
from email.mime.multipart import MIMEMultipart
 10 
from email.mime.multipart import MIMEMultipart
11 
 
 - 
 
- 
 
 11 
from dtr.utils.utils import fetchResponseUsingProxy
12 
 
 12 
 
13 
con = None
 13 
con = None
14 
parser = optparse.OptionParser()
 14 
parser = optparse.OptionParser()
15 
parser.add_option("-m", "--m", dest="mongoHost",
 15 
parser.add_option("-m", "--m", dest="mongoHost",
16 
                      default="localhost",
 16 
                      default="localhost",
Line 35... Line 35...
35 
    if con is None:
 35 
    if con is None:
36 
        print "Establishing connection %s host and port %d" %(host,port)
 36 
        print "Establishing connection %s host and port %d" %(host,port)
37 
        try:
 37 
        try:
38 
            con = pymongo.MongoClient(host, port)
 38 
            con = pymongo.MongoClient(host, port)
39 
        except Exception, e:
 39 
        except Exception, e:
- 
 
 40 
 
40 
            print e
 41 
            print e
41 
            return None
 42 
            return None
42 
    return con
 43 
    return con
43 
 
 44 
 
44 
def getSoupObject(url):
 45 
def getSoupObject(url):
45 
    print "Getting soup object for"
 46 
    print "Getting soup object for"
46 
    print url
 47 
    print url
47 
    global RETRY_COUNT
 - 
 
48 
    RETRY_COUNT = 1
- 
 
49 
    while RETRY_COUNT < 10:
 - 
 
50 
        try:
 - 
 
51 
            soup = None
 48 
    soup = None
52 
            request = urllib2.Request(url)
 - 
 
53 
            request.add_header('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8')
 - 
 
54 
            request.add_header('Accept-Language','en-US,en;q=0.8,hi;q=0.6')
 - 
 
55 
            request.add_header('Connection','keep-alive')
 - 
 
56 
            request.add_header('User-Agent','Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36')
 - 
 
57
- 
 
58 
            response = urllib2.urlopen(request)
- 
 
59 
            response_data = response.read()
 49 
    response_data = fetchResponseUsingProxy(url)
60 
            response.close()
 50 
    print response_data
61 
            try:
 51 
    try:
62 
                page=response_data.decode("utf-8")
 52 
        page=response_data.decode("utf-8")
63 
                soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
 53 
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
64 
            except:
 54 
    except:
65 
                soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
 55 
        soup = BeautifulSoup(response_data,convertEntities=BeautifulSoup.HTML_ENTITIES)
66 
            if soup is None:
 - 
 
67 
                raise
 - 
 
68 
            return soup
 56 
    return soup
69 
        except Exception as e:
 - 
 
70 
            print e
 - 
 
71 
            print "Retrying"
 - 
 
72 
            RETRY_COUNT = RETRY_COUNT + 1
 - 
 
73 
 
 - 
 
74 
 
 57 
 
75 
def scrapeBestSellerMobiles():
 58 
def scrapeBestSellerMobiles():
76 
    global bestSellers
 59 
    global bestSellers
77 
    rank = 0
 60 
    rank = 0
78 
    for i in [1, 21, 41, 61, 81]:
 61 
    for i in [1, 21, 41, 61, 81]:
Line 140... Line 123...
140 
        <td style="text-align:center">"""+(item.category)+"""</td>
 123 
        <td style="text-align:center">"""+(item.category)+"""</td>
141 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
 124 
        <td style="text-align:center">"""+str(item.rank)+"""</td>
142 
        </tr>"""
 125 
        </tr>"""
143 
    message+="""</tbody></table></body></html>"""
 126 
    message+="""</tbody></table></body></html>"""
144 
    print message
 127 
    print message
145 
    #recipients = ['kshitij.sood@saholic.com']
 128 
    recipients = ['kshitij.sood@saholic.com']
146 
    recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
 129 
    #recipients = ['rajneesh.arora@saholic.com','kshitij.sood@saholic.com','chaitnaya.vats@saholic.com','manoj.kumar@saholic.com']
147 
    msg = MIMEMultipart()
 130 
    msg = MIMEMultipart()
148 
    msg['Subject'] = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
 131 
    msg['Subject'] = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
149 
    msg['From'] = ""
 132 
    msg['From'] = ""
150 
    msg['To'] = ",".join(recipients)
 133 
    msg['To'] = ",".join(recipients)
151 
    msg.preamble = "Flipkart Best Sellers" + ' - ' + str(datetime.now())
 134 
    msg.preamble = "Flipkart Best Sellers" + ' - ' + str(datetime.now())