Subversion Repositories SmartDukaan

from elixir import *

from shop2020.config.client.ConfigClient import ConfigClient

from shop2020.clients.CatalogClient import CatalogClient

from shop2020.model.v1.catalog.impl import DataService

from shop2020.model.v1.catalog.impl.DataService import PrivateDealsPriceComparison, PrivateDeals, Amazonlisted, \

SnapdealItem, FlipkartItem

from shop2020.model.v1.catalog.script import FlipkartScraper, AmazonScraper

from operator import itemgetter

import urllib2

import time

import simplejson as json

from datetime import datetime

config_client = ConfigClient()

host = config_client.get_property('staging_hostname')

DataService.initialize(db_hostname=host)

scrapedInfo = []

scraperFk = FlipkartScraper.FlipkartScraper()

scraperAmazon = AmazonScraper.AmazonScraper()

class __ScrapingInfo:

    def __init__(self,itemId, dealPrice, saholicPrice, sdPrice, fkPrice, amazonPrice, supc, fsn, asin):

        self.itemId = itemId

        self.dealPrice = dealPrice

        self.saholicPrice = saholicPrice

        self.sdPrice = sdPrice

        self.fkPrice = fkPrice

        self.amazonPrice = amazonPrice

        self.supc = supc

        self.fsn = fsn

        self.asin = asin

def getAllActivePDFromMaster():

    catalog_client = CatalogClient().get_client()

    allActivePrivateDeals = catalog_client.getAllActivePrivateDeals(None, 0)

    return allActivePrivateDeals

def scrapeSnapdeal():

    for data in scrapedInfo:

        if data.supc is None or len(data.supc)==0:

            continue

        try:

            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=91&sort=sellingPrice"%(data.supc)

            print url

            time.sleep(1)

            req = urllib2.Request(url)

            response = urllib2.urlopen(req)

            json_input = response.read()

            vendorInfo = json.loads(json_input)

            lowestOfferPrice, iterator = (0,)*2

            for vendor in vendorInfo:

                if iterator == 0:

                    lowestOfferPrice = vendor['sellingPrice']

                break

            data.sdPrice = lowestOfferPrice

        except:

            continue

def scrapeFlipkart():

    for data in scrapedInfo:

        if data.fsn is None or len(data.fsn)==0:

            continue

        try:

            url = "http://www.flipkart.com/ps/%s"%(data.fsn)

            vendorsData = scraperFk.read(url)

            sortedVendorsData = []

            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))

            lowestSp, iterator = (0,)*2

            for vData in sortedVendorsData:

                if iterator == 0:

                    lowestSp = vData['sellingPrice']

                    data.fkPrice = lowestSp

                break

        except:

            continue    

def scrapeAmazon():

    for data in scrapedInfo:

        if data.asin is None or len(data.asin)==0:

            continue

        try:

            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(data.asin)

            scraperAmazon.read(url)

            lowestSp = scraperAmazon.createData()

            data.amazonPrice = lowestSp

        except:

            continue

def populateScrapingInfo():

    for data in scrapedInfo:

        amazon = Amazonlisted.get_by(itemId=data.itemId)

        snapdeal = SnapdealItem.get_by(item_id=data.itemId)

        flipkart = FlipkartItem.get_by(item_id=data.itemId)

        if amazon is not None:

            data.asin = amazon.asin

        if snapdeal is not None:

            data.supc = snapdeal.supc

        if flipkart is not None:

            data.fsn = flipkart.flipkartSerialNumber

def populateOurPrices():

    catalog_client = CatalogClient().get_client()

    for data in scrapedInfo:

        cat_item = catalog_client.getItem(data.itemId)

        data.saholicPrice = cat_item.sellingPrice

def commitData():

    PrivateDealsPriceComparison.query.delete()

    session.commit()

    for data in scrapedInfo:

        pdComp = PrivateDealsPriceComparison()

        pdComp.item_id = data.itemId

        pdComp.dealPrice = data.dealPrice

        pdComp.saholicPrice = data.saholicPrice

        pdComp.sdPrice = data.sdPrice

        pdComp.fkPrice = data.fkPrice

        pdComp.amazonPrice = data.amazonPrice

        pdComp.asin = data.asin

        pdComp.fsn = data.fsn

        pdComp.supc = data.supc

        pdComp.lastProcessedTimestamp = datetime.now()

    session.commit()

def main():

    privateDeals = getAllActivePDFromMaster()

    global scrapedInfo

    for itemId in privateDeals.iterkeys():

        temp = __ScrapingInfo(itemId,privateDeals.get(itemId).dealPrice,None,None,None,None,None,None,None )

        scrapedInfo.append(temp)

    privateDeals = {}

    populateScrapingInfo()

    scrapeSnapdeal()

    scrapeFlipkart()

    scrapeAmazon()

    populateOurPrices()

    commitData()

if __name__=='__main__':

    main()