Subversion Repositories SmartDukaan

from elixir import session

from shop2020.config.client.ConfigClient import ConfigClient

from sqlalchemy.sql import asc

from sqlalchemy.sql.expression import or_

from shop2020.utils.daemon import Daemon

import optparse

import sys

import threading

import urllib2

import mechanize

import cookielib

import time

import requests as httpRequest

import simplejson as json

from shop2020.model.v1.catalog.impl import DataService

from shop2020.model.v1.catalog.script import FlipkartScraper, AmazonScraper, SellerCentralScraper

from operator import itemgetter

from shop2020.model.v1.catalog.impl.DataService import CompetitorPricing, CompetitorPricingRequest, \

SnapdealItem, FlipkartItem

import gc 

config_client = ConfigClient()

host = config_client.get_property('staging_hostname')

DataService.initialize(db_hostname=host)

class CompetitorScraping(Daemon):

    def __init__(self, logfile='/var/log/services/competitorScraping.log', pidfile='/var/run/competitor-scraper.pid'):

        Daemon.__init__(self, pidfile, stdout=logfile, stderr=logfile)

    def run(self):

        start()

def start():

    try:

        while True:

            requests = session.query(CompetitorPricingRequest).filter(or_(CompetitorPricingRequest.isProcessed==False,CompetitorPricingRequest.isProcessed==None)).order_by(asc(CompetitorPricingRequest.requestId)).all()

            if requests ==[] or requests is None:

                print "No new request to process, sleeeeeeping....."

                close_session()

                collected = gc.collect()

                print locals()

                print globals()

                print "Garbage collector: collected %d objects." % (collected)

                time.sleep(600)

            for request in requests:

                fetchDetails(request)

                request.isProcessed = True

                session.commit()

            close_session()

            collected = gc.collect()

            print "Garbage collector: collected %d objects." % (collected)

            print locals()

            print globals()

            requests = []

    except Exception as e:

        print e

        sys.exit(2)

def fetchDetails(request):

    login_url = "https://sellercentral.amazon.in/gp/homepage.html"

    br = login(login_url)

    items = session.query(CompetitorPricing).filter(CompetitorPricing.competitorPricing_requestId==request.requestId).all()

    print items

    snapdeal, flipkart, amazon =[],[],[]

    for item in items:

        if item.snapdealScraping:

            snapdeal.append(item)

        if item.flipkartScraping:

            flipkart.append(item)

        if item.amazonScraping:

            amazon.append(item)

    threads = []

    t1 = threading.Thread(target=scrapSnapdeal, args = (snapdeal,))

    t1.daemon = True

    t1.start()

    t2 = threading.Thread(target=scrapFlipkart, args = (flipkart,))

    t2.daemon = True

    t2.start()

    t3 = threading.Thread(target=scrapAmazon, args = (amazon,br))

    t3.daemon = True

    t3.start()

    threads.append(t1)

    threads.append(t2)

    threads.append(t3)

    for th in threads:

        th.join()

    br,t1,t2,t3 =None,None,None,None

    items[:],snapdeal[:],flipkart[:],amazon[:],threads[:]=[],[],[],[],[]

def scrapSnapdeal(snapdealItems):

    for snapdealItem in snapdealItems:

        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)

        if sdItem is None:

            continue

        try:

            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=91&sort=sellingPrice"%(sdItem.supc)

            print url

            time.sleep(1)

            req = urllib2.Request(url)

            response = urllib2.urlopen(req)

            json_input = response.read()

            vendorInfo = json.loads(json_input)

            lowestSp, iterator, ourInventory, lowestSellerInventory,ourSp,ourOfferPrice,lowestSp,lowestOfferPrice   = (0,)*8

            lowestSellerName = ''

            for vendor in vendorInfo:

                if iterator == 0:

                    lowestSellerName = vendor['vendorDisplayName']

                    try:

                        lowestSp = vendor['sellingPriceBefIntCashBack']

                    except:

                        lowestSp = vendor['sellingPrice']

                    lowestOfferPrice = vendor['sellingPrice']

                    lowestSellerInventory = vendor['buyableInventory']

                if vendor['vendorDisplayName'] == 'MobilesnMore':

                    ourInventory = vendor['buyableInventory']

                    try:

                        ourSp = vendor['sellingPriceBefIntCashBack']

                    except:

                        ourSp = vendor['sellingPrice']

                    ourOfferPrice = vendor['sellingPrice']

                iterator+=1

        except:

            continue

        finally:

            sdItem =None

        snapdealItem.ourSnapdealPrice = ourSp

        snapdealItem.ourSnapdealOfferPrice = ourOfferPrice

        snapdealItem.ourSnapdealInventory = ourInventory

        snapdealItem.lowestSnapdealPrice = lowestSp

        snapdealItem.lowestSnapdealOfferPrice = lowestOfferPrice

        snapdealItem.lowestSnapdealSeller = lowestSellerName 

        snapdealItem.lowestSnapdealSellerInventory = lowestSellerInventory  

def scrapFlipkart(flipkartItems):

    scraperFk = FlipkartScraper.FlipkartScraper()

    for flipkartItem in flipkartItems:

        fkItem = FlipkartItem.get_by(item_id=flipkartItem.item_id)

        if fkItem is None:

            continue

        try:

            url = "http://www.flipkart.com/ps/%s"%(fkItem.flipkartSerialNumber)

            vendorsData = scraperFk.read(url)

            sortedVendorsData = []

            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))

            lowestSellerSp, iterator, ourSp = (0,)*3

            lowestSellerName = ''

            for data in sortedVendorsData:

                if iterator == 0:

                    lowestSellerName = data['sellerName']

                    lowestSellerSp = data['sellingPrice']

                if data['sellerName'] == 'Saholic':

                    ourSp = data['sellingPrice']

                iterator+=1

        except:

            continue

        finally:

            fkItem=None

        try:

            request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(fkItem.flipkartSerialNumber))

            r = httpRequest.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))

            print "Inventory info",r.json()

            stock_count = int((r.json()['attributeValues'])['stock_count'])

        except:

            stock_count = 0

        finally:

                r={}

        flipkartItem.ourFlipkartPrice = ourSp

        flipkartItem.ourFlipkartInventory = stock_count

        flipkartItem.lowestFlipkartPrice = lowestSellerSp

        flipkartItem.lowestFlipkartSeller =  lowestSellerName

    scraperFk = None

def close_session():

    if session.is_active:

        print "session is active. closing it."

        session.close()

def scrapAmazon(amazonItems,br):

    print "Inside amazonitems ",amazonItems

    print "len amazon items ",len(amazonItems)

    time.sleep(5)

    sc = SellerCentralScraper.SellerCentralScraper()

    scraperAmazon = AmazonScraper.AmazonScraper()

    for amazonItem in amazonItems:

        skuUrlMfn = "https://sellercentral.amazon.in/myi/search/ProductSummary?keyword=%d" %(amazonItem.item_id)

        skuUrlFba =  "https://sellercentral.amazon.in/myi/search/ProductSummary?keyword=FBA%d" %(amazonItem.item_id)

        try:

            asin, mfnInventory, mfnPrice= sc.requestSku(br, skuUrlMfn)

            fbaAsin, fbaInventory, fbaPrice= sc.requestSku(br, skuUrlFba)

        except Exception as e:

            print e

            print "Unable to fetch details from Seller Central for ",amazonItem.item_id

            continue

        try:

            if len(asin)==0 and len(fbaAsin)==0:

                print "No asin found for ",amazonItem.item_id

                continue

            if len(asin)==0:

                asin=fbaAsin

            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(asin)

            scraperAmazon.read(url,True)

            lowestSp,lowestSeller = scraperAmazon.createData()

            amazonItem.lowestAmazonPrice = lowestSp

            amazonItem.ourMfnPrice = float(str(mfnPrice).replace("Rs.","").replace(",",""))

            amazonItem.ourFbaPrice = float(str(fbaPrice).replace("Rs.","").replace(",",""))

            amazonItem.ourMfnInventory = int(mfnInventory)

            amazonItem.ourFbaInventory = int(fbaInventory)

            amazonItem.lowestAmazonPrice = float(lowestSp)

            amazonItem.lowestAmazonSeller = lowestSeller

        except Exception as e:

            print e

            print "Unable to fetch details from Amazon Listing page for ",amazonItem.item_id

            continue

    sc =None

    scraperAmazon = None

def getBrowserObject():

        br = mechanize.Browser(factory=mechanize.RobustFactory())

        cj = cookielib.LWPCookieJar()

        br.set_cookiejar(cj)

        br.set_handle_equiv(True)

        br.set_handle_redirect(True)

        br.set_handle_referer(True)

        br.set_handle_robots(False)

        br.set_debug_http(False)

        br.set_debug_redirects(False)

        br.set_debug_responses(False)

        br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)

        br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),

                         ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),

                         ('Accept-Encoding', 'gzip,deflate,sdch'),                  

                         ('Accept-Language', 'en-US,en;q=0.8'),                     

                         ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]

        return br

def ungzipResponse(r,b):

    headers = r.info()

    if headers['Content-Encoding']=='gzip':

        import gzip

        print "********************"

        print "Deflating gzip response"

        print "********************"

        gz = gzip.GzipFile(fileobj=r, mode='rb')

        html = gz.read()

        gz.close()

        headers["Content-type"] = "text/html; charset=utf-8"

        r.set_data( html )

        b.set_response(r)

def login(url):

    br = getBrowserObject()

    br.open(url)

    response = br.open(url)

    ungzipResponse(response, br)

    #html = response.read()

    #print html

    br.select_form(name="signinWidget")

    br.form['username'] = "kshitij.sood@saholic.com"

    br.form['password'] = "pioneer"

    response = br.submit()

    print "********************"

    print "Attempting to Login"

    print "********************"

    #ungzipResponse(response, br)

    return br

if __name__ == "__main__":

    parser = optparse.OptionParser()

    parser.add_option("-l", "--logfile", dest="logfile",

                      type="string",

                      help="Log all output to LOG_FILE",

                      )

    parser.add_option("-i", "--pidfile", dest="pidfile",

                      type="string",

                      help="Write the PID to pidfile")

    (options, args) = parser.parse_args()

    daemon = CompetitorScraping(options.logfile, options.pidfile)

    if len(args) == 0:

        daemon.run()

    elif len(args) == 1:

        if 'start' == args[0]:

            daemon.start()

        elif 'stop' == args[0]:

            daemon.stop()

        elif 'restart' == args[0]:

            daemon.restart()

        else:

            print "Unknown command"

            sys.exit(2)

        sys.exit(0)

    else:

        print "usage: %s start|stop|restart" % sys.argv[0]

        sys.exit(2)