Subversion Repositories SmartDukaan

from elixir import *
from shop2020.config.client.ConfigClient import ConfigClient
from shop2020.clients.CatalogClient import CatalogClient
from shop2020.model.v1.catalog.impl import DataService
from shop2020.model.v1.catalog.impl.DataService import PrivateDealsPriceComparison, PrivateDeals, Amazonlisted, \
SnapdealItem, FlipkartItem
from shop2020.model.v1.catalog.script import FlipkartScraper, AmazonScraper
from operator import itemgetter
import urllib2
import time
import simplejson as json
from datetime import datetime


config_client = ConfigClient()
host = config_client.get_property('staging_hostname')
DataService.initialize(db_hostname=host)

scrapedInfo = []
scraperFk = FlipkartScraper.FlipkartScraper()
scraperAmazon = AmazonScraper.AmazonScraper()

class __ScrapingInfo:
    def __init__(self,itemId, dealPrice, saholicPrice, sdPrice, fkPrice, amazonPrice, supc, fsn, asin):
        self.itemId = itemId
        self.dealPrice = dealPrice
        self.saholicPrice = saholicPrice
        self.sdPrice = sdPrice
        self.fkPrice = fkPrice
        self.amazonPrice = amazonPrice
        self.supc = supc
        self.fsn = fsn
        self.asin = asin

def getAllActivePDFromMaster():
    catalog_client = CatalogClient().get_client()
    allActivePrivateDeals = catalog_client.getAllActivePrivateDeals(None, 0)
    return allActivePrivateDeals

def scrapeSnapdeal():
    for data in scrapedInfo:
        if data.supc is None or len(data.supc)==0:
            continue
        try:
            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=175&sort=sellingPrice"%(data.supc)
            print url
            time.sleep(1)
            req = urllib2.Request(url)
            response = urllib2.urlopen(req)
            json_input = response.read()
            vendorInfo = json.loads(json_input)
            lowestOfferPrice, iterator = (0,)*2
            for vendor in vendorInfo:
                if iterator == 0:
                    lowestOfferPrice = vendor['sellingPrice']
                break
            data.sdPrice = lowestOfferPrice
        except:
            continue

def scrapeFlipkart():
    for data in scrapedInfo:
        if data.fsn is None or len(data.fsn)==0:
            continue
        try:
            url = "http://www.flipkart.com/ps/%s"%(data.fsn)
            vendorsData = scraperFk.read(url)
            sortedVendorsData = []
            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
            lowestSp, iterator = (0,)*2
            for vData in sortedVendorsData:
                if iterator == 0:
                    lowestSp = vData['sellingPrice']
                    data.fkPrice = lowestSp
                break
        except:
            continue    

def scrapeAmazon():
    for data in scrapedInfo:
        if data.asin is None or len(data.asin)==0:
            continue
        try:
            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data.asin)
            scraperAmazon.read(url,False)
            lowestSp,store = scraperAmazon.createData()
            data.amazonPrice = lowestSp
        except:
            continue

def populateScrapingInfo():
    for data in scrapedInfo:
        amazon = Amazonlisted.get_by(itemId=data.itemId)
        snapdeal = SnapdealItem.get_by(item_id=data.itemId)
        flipkart = FlipkartItem.get_by(item_id=data.itemId)
        if amazon is not None:
            data.asin = amazon.asin
        if snapdeal is not None:
            data.supc = snapdeal.supc
        if flipkart is not None:
            data.fsn = flipkart.flipkartSerialNumber

def populateOurPrices():
    catalog_client = CatalogClient().get_client()
    for data in scrapedInfo:
        cat_item = catalog_client.getItem(data.itemId)
        data.saholicPrice = cat_item.sellingPrice

def commitData():
    PrivateDealsPriceComparison.query.delete()
    session.commit()
    for data in scrapedInfo:
        pdComp = PrivateDealsPriceComparison()
        pdComp.item_id = data.itemId
        pdComp.dealPrice = data.dealPrice
        pdComp.saholicPrice = data.saholicPrice
        pdComp.sdPrice = data.sdPrice
        pdComp.fkPrice = data.fkPrice
        pdComp.amazonPrice = data.amazonPrice
        pdComp.asin = data.asin
        pdComp.fsn = data.fsn
        pdComp.supc = data.supc
        pdComp.lastProcessedTimestamp = datetime.now()
    session.commit()
      

def main():
    privateDeals = getAllActivePDFromMaster()
    global scrapedInfo
    for itemId in privateDeals.iterkeys():
        temp = __ScrapingInfo(itemId,privateDeals.get(itemId).dealPrice,None,None,None,None,None,None,None )
        scrapedInfo.append(temp)
    privateDeals = {}
    populateScrapingInfo()
    scrapeSnapdeal()
    scrapeFlipkart()
    scrapeAmazon()
    populateOurPrices()
    commitData()
    
    

if __name__=='__main__':
    main()