Subversion Repositories SmartDukaan

'''

Created on 31-Aug-2011

@author: Varun Gupta

'''

import tornado.httpserver, tornado.ioloop, tornado.web

import json, os, ConfigParser, sys

from PyLucene.Retriever import Retriever

from Utils import getItemsWithTopScore, isPriceSame, getProductClusters, getFilteredClustersWithTopScores, \

    getDisplayInfo, getValidItems, getProductName

from ScraperLoader import getScraper

from PyLucene.IndexBuilder import IndexBuilder

from DataStore.WatchListManager import WatchListManager

cmd_folder = os.path.dirname(os.path.abspath(os.environ["HOME"] + "/code/trunk/PyProj/src/shop2020/"))

if cmd_folder not in sys.path:

    sys.path.insert(0, cmd_folder)

from shop2020.clients.CatalogClient import CatalogClient

from shop2020.thriftpy.model.v1.catalog.ttypes import status

class BaseHandler(tornado.web.RequestHandler):

    def get_current_user(self):

        return self.get_secure_cookie('userauth')

class LoginHandler(BaseHandler):

    def get(self):

        self.loader = tornado.template.Loader('HTMLTemplates')

        self.write(self.loader.load('LoginForm.html').generate())

    def post(self):

        config = ConfigParser.SafeConfigParser()

        config.read('app.cfg')

        username = self.get_argument('username')

        password = self.get_argument('password')

        if username == config.get('auth', 'username') and password == config.get('auth', 'password'):

            print 'Password Matched'

            self.set_secure_cookie("userauth", username + '_' + password)

            self.redirect('/')

        else:

            self.redirect('/login')

class URLFeedbackHandler(BaseHandler):

    url_feedback_file = '/usr/price-comp-dashboard/urls.json'

    secondary_crawl_file = '/usr/price-comp-dashboard/secondary-crawl.json'

    def post(self):

        try:

            fp_read = open(URLFeedbackHandler.url_feedback_file, 'r')

            urls = json.load(fp_read)

        except ValueError as e:

            print e

            urls = {}

        finally:

            fp_read.close()

        print 'Existing URLs: ', urls

        entity = self.get_argument('entity')

        source = self.get_argument('source')

        url = self.get_argument('url')

        if entity in urls:

            urls[entity][source] = url

        else:

            urls[entity] = {source: url}

        print 'New set of URLs: ', urls

        fp_write = open(URLFeedbackHandler.url_feedback_file, 'w')

        json.dump(urls, fp_write, indent = 4)

        fp_write.close()

        #Scraping the page

        scraper = getScraper(source)

        productData = scraper.getDataFromProductPage(url)

        #Storing the data

        try:

            fp_read = open(URLFeedbackHandler.secondary_crawl_file, 'r')

            data = json.load(fp_read)

        except ValueError as e:

            print e

            data = {}

        finally:

            fp_read.close()

        if entity in data:

            data[entity][source] = productData

        else:

            data[entity] = {source: productData}

        print 'Secondary crawled data:', data

        fp_write = open(URLFeedbackHandler.secondary_crawl_file, 'w')

        json.dump(data, fp_write, indent = 4)

        fp_write.close()

        productData['entityId'] = entity

        self.write(productData)

    def get(self):

        try:

            fp_read = open(URLFeedbackHandler.secondary_crawl_file, 'r')

            data = json.load(fp_read)

        except ValueError as e:

            print e

            data = {}

        finally:

            fp_read.close()

        self.write(data)

class FeedbackHandler(BaseHandler):

    def save(self, entity, source, feedback_type, selected_item = None):

        self.feedback_file = '/usr/price-comp-dashboard/feedback.json'

        file_to_read = open(self.feedback_file, 'r')

        feedbacks_json = file_to_read.read()

        file_to_read.close()

        feedbacks = json.loads(feedbacks_json) if len(feedbacks_json) > 1 else {}

        if entity not in feedbacks: feedbacks[entity] = {}

        feedbacks[entity][source] = {'type': feedback_type}

        if selected_item is not None:   feedbacks[entity][source]['selected_item'] = selected_item

        file_to_write = open(self.feedback_file, 'w')

        json.dump(feedbacks, file_to_write, indent = 4)

        file_to_write.close()

    def post(self):

        feedback_type = self.get_argument('type')

        entity_id = self.get_argument('entityId')

        price_data_source = self.get_argument('source')

        print feedback_type, entity_id, price_data_source

        if feedback_type == 'select':

            selected_item = self.get_argument('selected')

            print selected_item

            self.save(entity_id, price_data_source, feedback_type, selected_item)

        else:

            self.save(entity_id, price_data_source, feedback_type)

    def get(self):

        print 'GET: Feedback data'

        self.feedback_file = '/usr/price-comp-dashboard/feedback.json'

        file_to_read = open(self.feedback_file, 'r')

        feedbacks_json = file_to_read.read()

        file_to_read.close()

        self.write(feedbacks_json)

class MainHandler(BaseHandler):

    def mapSearchUrls(self, map, name):

        search_urls = {

            'flipkart': 'http://www.flipkart.com/search/a/all?query=$$',

            'homeshop18': 'http://www.homeshop18.com/search:$$',

            'adexmart': 'http://adexmart.com/search.php?orderby=position&orderway=desc&search_query=$$',

            'infibeam': 'http://www.infibeam.com/search?q=$$',

            'snapdeal': '$$',

            'sulekha': 'http://mobiles.sulekha.com/search.htm?cx=partner-pub-3470583419345383%3A8ggsimfcaaa&cof=FORID%3A10&ie=ISO-8859-1&q=$$&sa=Go',

            'tradus': 'http://www.tradus.com/search/tradus_search/?query=$$'

        }

        for key in search_urls.iterkeys():

            try:

                if map[key]['url'] == 'Not Found':

                    map[key]['url'] = search_urls[key].replace('$$', name)

            except KeyError:

                map[key] = {'price': 'Not Found', 'url': search_urls[key].replace('$$', name)}

        return map

    @tornado.web.authenticated

    def get(self):

        self.loader = tornado.template.Loader('HTMLTemplates')

        catalog_client = CatalogClient().get_client()

        items = catalog_client.getAllItemsByStatus(status.ACTIVE)

        items.extend(catalog_client.getAllItemsByStatus(status.PAUSED))

        items.extend(catalog_client.getAllItemsByStatus(status.PAUSED_BY_RISK))

#        synonyms = getSynonyms()

#        print synonyms

        retriever = Retriever()

        products = {}

        for item in items:

            if item.category in (10002, 10003, 10004, 10005, 10010, 11002, 11003):  products[item.catalogItemId] = item

        comparative_prices = []

        for item in sorted(products.itervalues(), key = lambda item: item.brand):

            try:

                model_name = item.modelName.strip() if len(item.modelName.strip()) > 0 else None

                model_number = item.modelNumber.strip() if len(item.modelNumber.strip()) > 0 else None

                #synonyms_for_this_model = synonyms[item.catalogItemId] if item.catalogItemId in synonyms else None

                search_results = retriever.retrieve(model_number = model_number, model_name = model_name, brand = item.brand, synonyms = None)

                clusters = getProductClusters(search_results)

                filtered_clusters = getFilteredClustersWithTopScores(clusters)

                product_name = "%s " % item.brand

                product_name += "%s " % model_name if model_name is not None else ''

                product_name += model_number if model_number is not None else ''

                display_info = getDisplayInfo(filtered_clusters, product_name)

                print 'Display Info: ', display_info

                display_info['entity_id'] = item.catalogItemId

                display_info['product_name'] = product_name

                display_info['saholic'] = {'price': item.sellingPrice}

                comparative_prices.append(display_info)

            except Exception as e:

                print 'Exception for %s:' % item.catalogItemId, e

        json.dump(comparative_prices, open('/usr/pcd_log', 'w'), indent = 4)

        self.write(self.loader.load('PriceChart.html').generate(data = comparative_prices))

class WatchlistHandler(BaseHandler):

    @tornado.web.authenticated

    def get(self):

        watchlistManager = WatchListManager()

        watchlist = watchlistManager.getWatchlist()

        print 'Getting watchlist: ', watchlist

        entityIds = []

        for id in watchlist:

            entityIds.append(int(id))

        self.write(str(entityIds))

    def post(self):

        watchlistManager = WatchListManager()

        requestType = self.get_argument('type').strip()

        entityId = self.get_argument('entity')

        print 'Request Type:', requestType, ', Entity Id: ', entityId

        if requestType == 'save':

            watchlistManager.save(entity = entityId)

        elif requestType == 'delete':

            watchlistManager.remove(entity = entityId)

        self.write("{}")

class DownloadHandler(BaseHandler):

    def post(self):

        catalog_client = CatalogClient().get_client()

        retriever = Retriever()

        vendors = {}

        for vendor in catalog_client.getAllVendors():

            vendors[vendor.id] = vendor.name

        self.set_header('Content-Type', 'text/csv')

        self.set_header("Content-disposition", "inline; filename=price-comparison.xls")

        newLine = '\n'

        tab = '\t'

        header = 'Product' + tab

        header += 'Vendor' + tab + 'TP' + tab + 'Vendor' + tab + 'TP' + tab +  'Vendor' + tab + 'TP' + tab

        header += 'Saholic' + tab + 'Flipkart' + tab + 'Homeshop18' + tab + 'Infibeam' + tab +  'Snapdeal' + newLine

        responseText = header

        for item in getValidItems():

            vendorItemPricings = catalog_client.getAllItemPricing(item.id)

            sortedPricings = sorted(vendorItemPricings, key = lambda vendorItemPricing: vendorItemPricing.transferPrice)

            productName = getProductName(item)

            row = productName + tab

            if len(sortedPricings) > 0:

                row += vendors[sortedPricings[0].vendorId] + tab + str(sortedPricings[0].transferPrice) + tab

            else:

                row += tab + tab

            if len(sortedPricings) > 1:

                row += vendors[sortedPricings[1].vendorId] + tab + str(sortedPricings[1].transferPrice) + tab

            else:

                row += tab + tab

            if len(sortedPricings) > 2:

                row += vendors[sortedPricings[2].vendorId] + tab + str(sortedPricings[2].transferPrice) + tab

            else:

                row += tab + tab

            row += str(item.sellingPrice) + tab

            model_name = item.modelName.strip() if len(item.modelName.strip()) > 0 else None

            model_number = item.modelNumber.strip() if len(item.modelNumber.strip()) > 0 else None

            search_results = retriever.retrieve(model_number = model_number, model_name = model_name, brand = item.brand, synonyms = None)

            clusters = getProductClusters(search_results)

            filtered_clusters = getFilteredClustersWithTopScores(clusters)

            display_info = getDisplayInfo(filtered_clusters, productName)

            if 'price' in display_info['flipkart'] and display_info['flipkart']['price'] is not None:

                row += display_info['flipkart']['price'] + tab

            else:

                row += tab

            if 'price' in display_info['homeshop18'] and display_info['homeshop18']['price'] is not None:

                row += display_info['homeshop18']['price'] + tab

            else:

                row += tab

            if 'price' in display_info['infibeam'] and display_info['infibeam']['price'] is not None:

                row += display_info['infibeam']['price'] + tab

            else:

                row += tab

            if 'price' in display_info['snapdeal'] and display_info['snapdeal']['price'] is not None:

                row += display_info['snapdeal']['price'] + tab

            else:

                row += tab

            if 'price' in display_info['sulekha'] and display_info['sulekha']['price'] is not None:

                row += display_info['sulekha']['price'] + tab

            else:

                row += tab

            if 'price' in display_info['tradus'] and display_info['tradus']['price'] is not None:

                row += display_info['tradus']['price'] + tab

            else:

                row += tab

            responseText += row + newLine

        self.write(responseText)

settings  = {

        'static_path': os.path.join(os.path.dirname(__file__), 'static'),

        'login_url': '/login', 

        'cookie_secret' :"61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo="

}

application = tornado.web.Application([

                (r"/", MainHandler),

                (r"/login", LoginHandler),

                (r"/feedback", FeedbackHandler),

                (r"/feedback-url", URLFeedbackHandler),

                (r"/watchlist", WatchlistHandler),

                (r"/download", DownloadHandler),

                (r"/(jquery-1.6.2.min\.js)", tornado.web.StaticFileHandler, dict(path=settings['static_path']))

            ], **settings)

if __name__ == '__main__':

    http_server = tornado.httpserver.HTTPServer(application)

    http_server.listen(8889)

    tornado.ioloop.IOLoop.instance().start()