Subversion Repositories SmartDukaan

from elixir import session
from shop2020.config.client.ConfigClient import ConfigClient
from sqlalchemy.sql import asc
from sqlalchemy.sql.expression import or_
from shop2020.utils.daemon import Daemon
import optparse
import sys
import threading
import urllib2
import mechanize
import cookielib
import time
import requests as httpRequest
import simplejson as json
from shop2020.model.v1.catalog.impl import DataService
from shop2020.model.v1.catalog.script import FlipkartScraper, AmazonScraper, SellerCentralScraper
from operator import itemgetter
from shop2020.model.v1.catalog.impl.DataService import CompetitorPricing, CompetitorPricingRequest, \
SnapdealItem, FlipkartItem
import gc 

config_client = ConfigClient()
host = config_client.get_property('staging_hostname')
DataService.initialize(db_hostname=host)

class CompetitorScraping(Daemon):
    def __init__(self, logfile='/var/log/services/competitorScraping.log', pidfile='/var/run/competitor-scraper.pid'):
        Daemon.__init__(self, pidfile, stdout=logfile, stderr=logfile)
        
    def run(self):
        start()

def start():
    try:
        while True:
            requests = session.query(CompetitorPricingRequest).filter(or_(CompetitorPricingRequest.isProcessed==False,CompetitorPricingRequest.isProcessed==None)).order_by(asc(CompetitorPricingRequest.requestId)).all()
            if requests ==[] or requests is None:
                print "No new request to process, sleeeeeeping....."
                close_session()
                collected = gc.collect()
                print locals()
                print globals()
                print "Garbage collector: collected %d objects." % (collected)
                time.sleep(600)
            for request in requests:
                fetchDetails(request)
                request.isProcessed = True
                session.commit()
            close_session()
            collected = gc.collect()
            print "Garbage collector: collected %d objects." % (collected)
            print locals()
            print globals()
            requests = []
    except Exception as e:
        print e
        sys.exit(2)
    
def fetchDetails(request):
    login_url = "https://sellercentral.amazon.in/gp/homepage.html"
    br = login(login_url)
    items = session.query(CompetitorPricing).filter(CompetitorPricing.competitorPricing_requestId==request.requestId).all()
    print items
    snapdeal, flipkart, amazon =[],[],[]
    for item in items:
        if item.snapdealScraping:
            snapdeal.append(item)
        if item.flipkartScraping:
            flipkart.append(item)
        if item.amazonScraping:
            amazon.append(item)
    threads = []
    t1 = threading.Thread(target=scrapSnapdeal, args = (snapdeal,))
    t1.daemon = True
    t1.start()
    t2 = threading.Thread(target=scrapFlipkart, args = (flipkart,))
    t2.daemon = True
    t2.start()
    t3 = threading.Thread(target=scrapAmazon, args = (amazon,br))
    t3.daemon = True
    t3.start()
    threads.append(t1)
    threads.append(t2)
    threads.append(t3)
    for th in threads:
        th.join()
    br,t1,t2,t3 =None,None,None,None
    items[:],snapdeal[:],flipkart[:],amazon[:],threads[:]=[],[],[],[],[]

def scrapSnapdeal(snapdealItems):
    for snapdealItem in snapdealItems:
        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)
        if sdItem is None:
            continue
        try:
            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=91&sort=sellingPrice"%(sdItem.supc)
            print url
            time.sleep(1)
            req = urllib2.Request(url)
            response = urllib2.urlopen(req)
            json_input = response.read()
            vendorInfo = json.loads(json_input)
            lowestSp, iterator, ourInventory, lowestSellerInventory,ourSp,ourOfferPrice,lowestSp,lowestOfferPrice   = (0,)*8
            lowestSellerName = ''
            for vendor in vendorInfo:
                if iterator == 0:
                    lowestSellerName = vendor['vendorDisplayName']
                    try:
                        lowestSp = vendor['sellingPriceBefIntCashBack']
                    except:
                        lowestSp = vendor['sellingPrice']
                    lowestOfferPrice = vendor['sellingPrice']
                    lowestSellerInventory = vendor['buyableInventory']
                    
                if vendor['vendorDisplayName'] == 'MobilesnMore':
                    ourInventory = vendor['buyableInventory']
                    try:
                        ourSp = vendor['sellingPriceBefIntCashBack']
                    except:
                        ourSp = vendor['sellingPrice']
                    ourOfferPrice = vendor['sellingPrice']
                iterator+=1
        except:
            continue
        finally:
            sdItem =None
        snapdealItem.ourSnapdealPrice = ourSp
        snapdealItem.ourSnapdealOfferPrice = ourOfferPrice
        snapdealItem.ourSnapdealInventory = ourInventory
        snapdealItem.lowestSnapdealPrice = lowestSp
        snapdealItem.lowestSnapdealOfferPrice = lowestOfferPrice
        snapdealItem.lowestSnapdealSeller = lowestSellerName 
        snapdealItem.lowestSnapdealSellerInventory = lowestSellerInventory  

def scrapFlipkart(flipkartItems):
    scraperFk = FlipkartScraper.FlipkartScraper()
    for flipkartItem in flipkartItems:
        fkItem = FlipkartItem.get_by(item_id=flipkartItem.item_id)
        if fkItem is None:
            continue
        try:
            url = "http://www.flipkart.com/ps/%s"%(fkItem.flipkartSerialNumber)
            vendorsData = scraperFk.read(url)
            sortedVendorsData = []
            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
            lowestSellerSp, iterator, ourSp = (0,)*3
            lowestSellerName = ''
            for data in sortedVendorsData:
                if iterator == 0:
                    lowestSellerName = data['sellerName']
                    lowestSellerSp = data['sellingPrice']
                        
                if data['sellerName'] == 'Saholic':
                    ourSp = data['sellingPrice']

                iterator+=1
        except:
            continue
        finally:
            fkItem=None
        try:
            request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(fkItem.flipkartSerialNumber))
            r = httpRequest.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
            print "Inventory info",r.json()
            stock_count = int((r.json()['attributeValues'])['stock_count'])
        except:
            stock_count = 0
        finally:
                r={}
        flipkartItem.ourFlipkartPrice = ourSp
        flipkartItem.ourFlipkartInventory = stock_count
        flipkartItem.lowestFlipkartPrice = lowestSellerSp
        flipkartItem.lowestFlipkartSeller =  lowestSellerName
    scraperFk = None


def close_session():
    if session.is_active:
        print "session is active. closing it."
        session.close()
        
def scrapAmazon(amazonItems,br):
    print "Inside amazonitems ",amazonItems
    print "len amazon items ",len(amazonItems)
    time.sleep(5)
    sc = SellerCentralScraper.SellerCentralScraper()
    scraperAmazon = AmazonScraper.AmazonScraper()
    for amazonItem in amazonItems:
        skuUrlMfn = "https://sellercentral.amazon.in/myi/search/ProductSummary?keyword=%d" %(amazonItem.item_id)
        skuUrlFba =  "https://sellercentral.amazon.in/myi/search/ProductSummary?keyword=FBA%d" %(amazonItem.item_id)
        try:
            asin, mfnInventory, mfnPrice= sc.requestSku(br, skuUrlMfn)
            fbaAsin, fbaInventory, fbaPrice= sc.requestSku(br, skuUrlFba)
        except Exception as e:
            print e
            print "Unable to fetch details from Seller Central for ",amazonItem.item_id
            continue
        try:
            if len(asin)==0 and len(fbaAsin)==0:
                print "No asin found for ",amazonItem.item_id
                continue
            if len(asin)==0:
                asin=fbaAsin
            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(asin)
            scraperAmazon.read(url,True)
            lowestSp,lowestSeller = scraperAmazon.createData()
            amazonItem.lowestAmazonPrice = lowestSp
            amazonItem.ourMfnPrice = float(str(mfnPrice).replace("Rs.","").replace(",",""))
            amazonItem.ourFbaPrice = float(str(fbaPrice).replace("Rs.","").replace(",",""))
            amazonItem.ourMfnInventory = int(mfnInventory)
            amazonItem.ourFbaInventory = int(fbaInventory)
            amazonItem.lowestAmazonPrice = float(lowestSp)
            amazonItem.lowestAmazonSeller = lowestSeller

        except Exception as e:
            print e
            print "Unable to fetch details from Amazon Listing page for ",amazonItem.item_id
            continue
    sc =None
    scraperAmazon = None
        
def getBrowserObject():
        br = mechanize.Browser(factory=mechanize.RobustFactory())
        cj = cookielib.LWPCookieJar()
        br.set_cookiejar(cj)
        br.set_handle_equiv(True)
        br.set_handle_redirect(True)
        br.set_handle_referer(True)
        br.set_handle_robots(False)
        br.set_debug_http(False)
        br.set_debug_redirects(False)
        br.set_debug_responses(False)
        
        br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
        
        br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
                         ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
                         ('Accept-Encoding', 'gzip,deflate,sdch'),                  
                         ('Accept-Language', 'en-US,en;q=0.8'),                     
                         ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]
        return br

def ungzipResponse(r,b):
    headers = r.info()
    if headers['Content-Encoding']=='gzip':
        import gzip
        print "********************"
        print "Deflating gzip response"
        print "********************"
        gz = gzip.GzipFile(fileobj=r, mode='rb')
        html = gz.read()
        gz.close()
        headers["Content-type"] = "text/html; charset=utf-8"
        r.set_data( html )
        b.set_response(r)

    
def login(url):
    br = getBrowserObject()
    br.open(url)
    response = br.open(url)
    ungzipResponse(response, br)
    #html = response.read()
    #print html
    br.select_form(name="signinWidget")
    br.form['username'] = "kshitij.sood@saholic.com"
    br.form['password'] = "pioneer"
    response = br.submit()
    print "********************"
    print "Attempting to Login"
    print "********************"
    #ungzipResponse(response, br)
    return br

        

if __name__ == "__main__":
    parser = optparse.OptionParser()
    parser.add_option("-l", "--logfile", dest="logfile",
                      type="string",
                      help="Log all output to LOG_FILE",
                      )
    parser.add_option("-i", "--pidfile", dest="pidfile",
                      type="string",
                      help="Write the PID to pidfile")
    (options, args) = parser.parse_args()
    daemon = CompetitorScraping(options.logfile, options.pidfile)
    if len(args) == 0:
        daemon.run()
    elif len(args) == 1:
        if 'start' == args[0]:
            daemon.start()
        elif 'stop' == args[0]:
            daemon.stop()
        elif 'restart' == args[0]:
            daemon.restart()
        else:
            print "Unknown command"
            sys.exit(2)
        sys.exit(0)
    else:
        print "usage: %s start|stop|restart" % sys.argv[0]
        sys.exit(2)