Subversion Repositories SmartDukaan

Rev

Rev 3440 | Rev 5291 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 3440 Rev 4198
Line 1... Line 1...
1 
'''
 1 
'''
2 
Created on 31-Aug-2011
 2 
Created on 31-Aug-2011
3 
 
 3 
 
4 
@author: Varun Gupta
 4 
@author: Varun Gupta
5 
'''
 5 
'''
6 
import tornado.httpserver
 6 
import tornado.httpserver, tornado.ioloop, tornado.web
7 
import tornado.ioloop
 - 
 
8 
import tornado.web
 - 
 
9 
import json, os, ConfigParser, sys
 7 
import json, os, ConfigParser, sys
10 
from PyLucene.Retriever import Retriever
 8 
from PyLucene.Retriever import Retriever
11 
from Utils import getItemsWithTopScore, isPriceSame, getProductClusters, getFilteredClustersWithTopScores, \
 9 
from Utils import getItemsWithTopScore, isPriceSame, getProductClusters, getFilteredClustersWithTopScores, \
12 
    getDisplayInfo, getSynonyms
 10 
    getDisplayInfo, getSynonyms
- 
 
 11 
from ScraperLoader import getScraper
- 
 
 12 
from PyLucene.IndexBuilder import IndexBuilder
13 
 
 13 
 
14 
cmd_folder = os.path.dirname(os.path.abspath(os.environ["HOME"] + "/code/trunk/PyProj/src/shop2020/"))
 14 
cmd_folder = os.path.dirname(os.path.abspath(os.environ["HOME"] + "/code/trunk/PyProj/src/shop2020/"))
15 
if cmd_folder not in sys.path:
 15 
if cmd_folder not in sys.path:
16 
    sys.path.insert(0, cmd_folder)
 16 
    sys.path.insert(0, cmd_folder)
17 
 
 17 
 
Line 39... Line 39...
39 
            self.set_secure_cookie("userauth", username + '_' + password)
 39 
            self.set_secure_cookie("userauth", username + '_' + password)
40 
            self.redirect('/')
 40 
            self.redirect('/')
41 
        else:
 41 
        else:
42 
            self.redirect('/login')
 42 
            self.redirect('/login')
43 
 
 43 
 
- 
 
 44 
class URLFeedbackHandler(BaseHandler):
- 
 
 45
- 
 
 46 
    def post(self):
- 
 
 47 
        self.url_feedback_file = '/tmp/price-comp-dashboard/urls.json'
- 
 
 48
- 
 
 49 
        try:
- 
 
 50 
            fp_read = open(self.url_feedback_file, 'r')
- 
 
 51 
            urls = json.load(fp_read)
- 
 
 52
- 
 
 53 
        except ValueError as e:
- 
 
 54 
            print e
- 
 
 55 
            urls = {}
- 
 
 56 
        finally:
- 
 
 57 
            fp_read.close()
- 
 
 58 
        print urls
- 
 
 59
- 
 
 60 
        entity = self.get_argument('entity')
- 
 
 61 
        source = self.get_argument('source')
- 
 
 62 
        url = self.get_argument('url')
- 
 
 63
- 
 
 64 
        if entity in urls:
- 
 
 65 
            urls[entity][source] = url
- 
 
 66 
        else:
- 
 
 67 
            urls[entity] = {source: url}
- 
 
 68
- 
 
 69 
        fp_write = open(self.url_feedback_file, 'w')
- 
 
 70 
        json.dump(urls, fp_write, indent = 4)
- 
 
 71 
        fp_write.close()
- 
 
 72
- 
 
 73 
        #Scraping the page
- 
 
 74 
        scraper = getScraper(source)
- 
 
 75 
        data = scraper.getDataFromProductPage(url)
- 
 
 76 
        index_builder = IndexBuilder([data], new_index = False)
- 
 
 77 
        index_builder.build()
- 
 
 78 
        self.write(data)
- 
 
 79
- 
 
 80 
    def get(self):
- 
 
 81 
        self.url_feedback_file = '/tmp/price-comp-dashboard/urls.json'
- 
 
 82 
 
44 
class FeedbackHandler(BaseHandler):
 83 
class FeedbackHandler(BaseHandler):
45
84
46 
    def save(self, entity, source, feedback_type, selected_item = None):
 85 
    def save(self, entity, source, feedback_type, selected_item = None):
47 
        self.feedback_file = '/tmp/feedback.json'
 86 
        self.feedback_file = '/tmp/price-comp-dashboard/feedback.json'
48 
        file_to_read = open(self.feedback_file, 'r')
 87 
        file_to_read = open(self.feedback_file, 'r')
49
88
50 
        feedbacks_json = file_to_read.read()
 89 
        feedbacks_json = file_to_read.read()
51 
        file_to_read.close()
 90 
        file_to_read.close()
52
91
Line 76... Line 115...
76 
        else:
 115 
        else:
77 
            self.save(entity_id, price_data_source, feedback_type)
 116 
            self.save(entity_id, price_data_source, feedback_type)
78
117
79 
    def get(self):
 118 
    def get(self):
80 
        print 'GET: Feedback data'
 119 
        print 'GET: Feedback data'
81 
        self.feedback_file = '/tmp/feedback.json'
 120 
        self.feedback_file = '/tmp/price-comp-dashboard/feedback.json'
82 
        file_to_read = open(self.feedback_file, 'r')
 121 
        file_to_read = open(self.feedback_file, 'r')
83
122
84 
        feedbacks_json = file_to_read.read()
 123 
        feedbacks_json = file_to_read.read()
85 
        file_to_read.close()
 124 
        file_to_read.close()
86
125
87 
        self.write(feedbacks_json)
 126 
        self.write(feedbacks_json)
88
127
89 
class MainHandler(BaseHandler):
 128 
class MainHandler(BaseHandler):
90 
 
 129 
 
- 
 
 130 
    def mapSearchUrls(self, map, name):
- 
 
 131
- 
 
 132 
        search_urls = {
- 
 
 133 
            'flipkart': 'http://www.flipkart.com/search-mobiles?query=$$&from=all&searchGroup=mobiles',
- 
 
 134 
            'homeshop18': 'http://www.homeshop18.com/nokia%20n97/search:$$/categoryid:3024',
- 
 
 135 
            'adexmart': 'http://adexmart.com/search.php?orderby=position&orderway=desc&search_query=$$',
- 
 
 136 
            'infibeam': 'http://www.infibeam.com/Mobiles/search?q=$$',
- 
 
 137 
            'letsbuy': 'http://www.letsbuy.com/advanced_search_result.php?cPath=254&keywords=$$'
- 
 
 138 
        }
- 
 
 139
- 
 
 140 
        for key in search_urls.iterkeys():
- 
 
 141 
            try:
- 
 
 142 
                if map[key]['url'] == 'Not Found':
- 
 
 143 
                    map[key]['url'] = search_urls[key].replace('$$', name)
- 
 
 144 
            except KeyError:
- 
 
 145 
                map[key] = {'price': 'Not Found', 'url': search_urls[key].replace('$$', name)}
- 
 
 146 
        return map
- 
 
 147
- 
 
 148
91 
    @tornado.web.authenticated
 149 
    @tornado.web.authenticated
92 
    def get(self):
 150 
    def get(self):
93 
        self.loader = tornado.template.Loader('HTMLTemplates')
 151 
        self.loader = tornado.template.Loader('HTMLTemplates')
94 
        catalog_client = CatalogClient().get_client()
 152 
        catalog_client = CatalogClient().get_client()
95 
        items = catalog_client.getAllItemsByStatus(status.ACTIVE)
 153 
        items = catalog_client.getAllItemsByStatus(status.ACTIVE)
Line 100... Line 158...
100 
        retriever = Retriever()
 158 
        retriever = Retriever()
101 
        products = {}
 159 
        products = {}
102
160
103 
        for item in items:
 161 
        for item in items:
104 
            if item.category in (10002, 10003, 10004, 10005, 10010):  products[item.catalogItemId] = item
 162 
            if item.category in (10002, 10003, 10004, 10005, 10010):  products[item.catalogItemId] = item
105 
 
 163
106 
        comparative_prices = []
 164 
        comparative_prices = []
107 
 
 165 
 
108 
        for item in sorted(products.itervalues(), key = lambda item: item.brand):
 166 
        for item in sorted(products.itervalues(), key = lambda item: item.brand):
109 
            try:
 167 
            try:
110 
                model_name = item.modelName.strip() if len(item.modelName.strip()) > 0 else None
 168 
                model_name = item.modelName.strip() if len(item.modelName.strip()) > 0 else None
Line 115... Line 173...
115 
                search_results = retriever.retrieve(model_number = model_number, model_name = model_name, brand = item.brand, synonyms = synonyms_for_this_model)
 173 
                search_results = retriever.retrieve(model_number = model_number, model_name = model_name, brand = item.brand, synonyms = synonyms_for_this_model)
116
174
117 
                clusters = getProductClusters(search_results)
 175 
                clusters = getProductClusters(search_results)
118 
                filtered_clusters = getFilteredClustersWithTopScores(clusters)
 176 
                filtered_clusters = getFilteredClustersWithTopScores(clusters)
119
177
120 
                display_info = getDisplayInfo(filtered_clusters)
 - 
 
121
- 
 
122 
                product_name = "%s " % item.brand
 178 
                product_name = "%s " % item.brand
123 
                product_name += "%s " % model_name if model_name is not None else ''
 179 
                product_name += "%s " % model_name if model_name is not None else ''
124 
                product_name += model_number if model_number is not None else ''
 180 
                product_name += model_number if model_number is not None else ''
125
181
- 
 
 182 
                display_info = getDisplayInfo(filtered_clusters, product_name)
- 
 
 183
126 
                display_info['entity_id'] = item.catalogItemId
 184 
                display_info['entity_id'] = item.catalogItemId
127 
                display_info['product_name'] = product_name
 185 
                display_info['product_name'] = product_name
128 
                display_info['saholic'] = {'price': item.sellingPrice}
 186 
                display_info['saholic'] = {'price': item.sellingPrice}
129 
                comparative_prices.append(display_info)
 187 
                comparative_prices.append(display_info)
130 
            except Exception as e:
 188 
            except Exception as e:
131 
                print e
 189 
                print 'Exception:', e
132
190
133 
        self.write(self.loader.load('PriceChart.html').generate(data = comparative_prices))
 191 
        self.write(self.loader.load('PriceChart.html').generate(data = comparative_prices))
134 
 
 192 
 
135 
 
 193 
 
136 
settings  = {
 194 
settings  = {
Line 141... Line 199...
141 
 
 199 
 
142 
application = tornado.web.Application([
 200 
application = tornado.web.Application([
143 
                (r"/", MainHandler),
 201 
                (r"/", MainHandler),
144 
                (r"/login", LoginHandler),
 202 
                (r"/login", LoginHandler),
145 
                (r"/feedback", FeedbackHandler),
 203 
                (r"/feedback", FeedbackHandler),
- 
 
 204 
                (r"/feedback-url", URLFeedbackHandler),
146 
                (r"/(jquery-1.6.2.min\.js)", tornado.web.StaticFileHandler, dict(path=settings['static_path']))
 205 
                (r"/(jquery-1.6.2.min\.js)", tornado.web.StaticFileHandler, dict(path=settings['static_path']))
147 
            ], **settings)
 206 
            ], **settings)
148 
 
 207 
 
149 
if __name__ == '__main__':
 208 
if __name__ == '__main__':
150 
    http_server = tornado.httpserver.HTTPServer(application)
 209 
    http_server = tornado.httpserver.HTTPServer(application)