Subversion Repositories SmartDukaan

from elixir import session
from sqlalchemy.sql import asc
from sqlalchemy.sql.expression import or_
from shop2020.utils.daemon import Daemon
import optparse
import sys
import threading
import urllib2
import mechanize
import cookielib
import time
import requests as httpRequest
import simplejson as json
from shop2020.model.v1.catalog.impl import DataService
from shop2020.model.v1.catalog.script import FlipkartScraper, AmazonScraper, SellerCentralScraper
from operator import itemgetter
from shop2020.model.v1.catalog.impl.DataService import CompetitorPricing, CompetitorPricingRequest, \
SnapdealItem, FlipkartItem 

DataService.initialize(db_hostname='localhost')

class CompetitorScraping(Daemon):
    def __init__(self, logfile='/var/log/services/competitorScraping.log', pidfile='/var/run/competitor-scraper.pid'):
        Daemon.__init__(self, pidfile, stdout=logfile, stderr=logfile)
        
    def run(self):
        start()

def start():
    try:
        while True:
            requests = session.query(CompetitorPricingRequest).filter(or_(CompetitorPricingRequest.isProcessed==False,CompetitorPricingRequest.isProcessed==None)).order_by(asc(CompetitorPricingRequest.requestId)).all()
            if requests ==[] or requests is None:
                print "No new request to process, sleeeeeeping....."
                time.sleep(600)
            for request in requests:
                fetchDetails(request)
                request.isProcessed = True
                session.commit()
            close_session()
    except Exception as e:
        print e
        sys.exit(2)
    
def fetchDetails(request):
    login_url = "https://sellercentral.amazon.in/gp/homepage.html"
    br = login(login_url)
    items = session.query(CompetitorPricing).filter(CompetitorPricing.competitorPricing_requestId==request.requestId).all()
    print items
    snapdeal, flipkart, amazon =[],[],[]
    for item in items:
        if item.snapdealScraping:
            snapdeal.append(item)
        if item.flipkartScraping:
            flipkart.append(item)
        if item.amazonScraping:
            amazon.append(item)
    threads = []
    t1 = threading.Thread(target=scrapSnapdeal, args = (snapdeal,))
    t1.daemon = True
    t1.start()
    t2 = threading.Thread(target=scrapFlipkart, args = (flipkart,))
    t2.daemon = True
    t2.start()
    amazonLen = len(amazon)
    if amazonLen > 20:
        t3 = threading.Thread(target=scrapAmazon, args = (amazon[0:amazonLen/2],br))
        t3.daemon = True
        t3.start()
        t4 = threading.Thread(target=scrapAmazon, args = (amazon[amazonLen/2:],br))
        t4.daemon = True
        t4.start()
        threads.append(t4)
    else:
        t3 = threading.Thread(target=scrapAmazon, args = (amazon,br))
        t3.daemon = True
        t3.start()
    threads.append(t1)
    threads.append(t2)
    threads.append(t3)
    for th in threads:
        th.join()

def scrapSnapdeal(snapdealItems):
    for snapdealItem in snapdealItems:
        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)
        if sdItem is None:
            continue
        try:
            url="http://www.snapdeal.com/acors/json/gvbps?supc=%s&catId=91&sort=sellingPrice"%(sdItem.supc)
            print url
            time.sleep(1)
            req = urllib2.Request(url)
            response = urllib2.urlopen(req)
            json_input = response.read()
            vendorInfo = json.loads(json_input)
            lowestSp, iterator, ourInventory, lowestSellerInventory,ourSp,ourOfferPrice,lowestSp,lowestOfferPrice   = (0,)*8
            lowestSellerName = ''
            for vendor in vendorInfo:
                if iterator == 0:
                    lowestSellerName = vendor['vendorDisplayName']
                    try:
                        lowestSp = vendor['sellingPriceBefIntCashBack']
                    except:
                        lowestSp = vendor['sellingPrice']
                    lowestOfferPrice = vendor['sellingPrice']
                    lowestSellerInventory = vendor['buyableInventory']
                    
                if vendor['vendorDisplayName'] == 'MobilesnMore':
                    ourInventory = vendor['buyableInventory']
                    try:
                        ourSp = vendor['sellingPriceBefIntCashBack']
                    except:
                        ourSp = vendor['sellingPrice']
                    ourOfferPrice = vendor['sellingPrice']
                iterator+=1
        except:
            continue
        snapdealItem.ourSnapdealPrice = ourSp
        snapdealItem.ourSnapdealOfferPrice = ourOfferPrice
        snapdealItem.ourSnapdealInventory = ourInventory
        snapdealItem.lowestSnapdealPrice = lowestSp
        snapdealItem.lowestSnapdealOfferPrice = lowestOfferPrice
        snapdealItem.lowestSnapdealSeller = lowestSellerName 
        snapdealItem.lowestSnapdealSellerInventory = lowestSellerInventory  

def scrapFlipkart(flipkartItems):
    for flipkartItem in flipkartItems:
        scraperFk = FlipkartScraper.FlipkartScraper()
        fkItem = FlipkartItem.get_by(item_id=flipkartItem.item_id)
        if fkItem is None:
            continue
        try:
            url = "http://www.flipkart.com/ps/%s"%(fkItem.flipkartSerialNumber)
            vendorsData = scraperFk.read(url)
            sortedVendorsData = []
            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
            lowestSellerSp, iterator, ourSp = (0,)*3
            lowestSellerName = ''
            for data in sortedVendorsData:
                if iterator == 0:
                    lowestSellerName = data['sellerName']
                    lowestSellerSp = data['sellingPrice']
                        
                if data['sellerName'] == 'Saholic':
                    ourSp = data['sellingPrice']

                iterator+=1
        except:
            continue
        
        try:
            request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(fkItem.flipkartSerialNumber))
            r = httpRequest.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
            print "Inventory info",r.json()
            stock_count = int((r.json()['attributeValues'])['stock_count'])
        except:
            stock_count = 0
        finally:
                r={}
        flipkartItem.ourFlipkartPrice = ourSp
        flipkartItem.ourFlipkartInventory = stock_count
        flipkartItem.lowestFlipkartPrice = lowestSellerSp
        flipkartItem.lowestFlipkartSeller =  lowestSellerName
        


def close_session():
    if session.is_active:
        print "session is active. closing it."
        session.close()
        
def scrapAmazon(amazonItems,br):
    for amazonItem in amazonItems:
        sc = SellerCentralScraper.SellerCentralScraper()
        skuUrlMfn = "https://sellercentral.amazon.in/myi/search/ProductSummary?keyword=%d" %(amazonItem.item_id)
        skuUrlFba =  "https://sellercentral.amazon.in/myi/search/ProductSummary?keyword=FBA%d" %(amazonItem.item_id)
        try:
            print type(sc.requestSku(br, skuUrlMfn))
            asin, mfnInventory, mfnPrice= sc.requestSku(br, skuUrlMfn)
            fbaAsin, fbaInventory, fbaPrice= sc.requestSku(br, skuUrlFba)
        except Exception as e:
            print e
            print "Unable to fetch details from Seller Central for ",amazonItem.item_id
            continue
        scraperAmazon = AmazonScraper.AmazonScraper()
        try:
            if len(asin)==0 and len(fbaAsin)==0:
                print "No asin found for ",amazonItem.item_id
                continue
            if len(asin)==0:
                asin=fbaAsin
            url = "http://www.amazon.in/gp/offer-listing/%s/ref=olp_sort_ps"%(asin)
            scraperAmazon.read(url,True)
            lowestSp,lowestSeller = scraperAmazon.createData()
            amazonItem.lowestAmazonPrice = lowestSp
            
            print "Details of Item id ",amazonItem.item_id
            print "ASIN ",asin
            print "MFN INVENTORY ",mfnInventory
            print "FBA INVENTORY ",fbaInventory
            print "MFN PRICE ",mfnPrice
            print "FBA PRICE ",fbaPrice
            print "Lowest Seller SP ",lowestSp
            print "LOWEST SELLER NAME ",lowestSeller
            
            amazonItem.ourMfnPrice = float(mfnPrice.replace("Rs.","").replace(",",""))
            amazonItem.ourFbaPrice = float(fbaPrice.replace("Rs.","").replace(",",""))
            amazonItem.ourMfnInventory = int(mfnInventory)
            amazonItem.ourFbaInventory = int(fbaInventory)
            amazonItem.lowestAmazonPrice = float(lowestSp)
            amazonItem.lowestAmazonSeller = lowestSeller

        except Exception as e:
            print e
            print "Unable to fetch details from Amazon Listing page for ",amazonItem.item_id
            continue
        
def getBrowserObject():
        br = mechanize.Browser(factory=mechanize.RobustFactory())
        cj = cookielib.LWPCookieJar()
        br.set_cookiejar(cj)
        br.set_handle_equiv(True)
        br.set_handle_redirect(True)
        br.set_handle_referer(True)
        br.set_handle_robots(False)
        br.set_debug_http(False)
        br.set_debug_redirects(False)
        br.set_debug_responses(False)
        
        br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
        
        br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
                         ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
                         ('Accept-Encoding', 'gzip,deflate,sdch'),                  
                         ('Accept-Language', 'en-US,en;q=0.8'),                     
                         ('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]
        return br

def ungzipResponse(r,b):
    headers = r.info()
    if headers['Content-Encoding']=='gzip':
        import gzip
        print "********************"
        print "Deflating gzip response"
        print "********************"
        gz = gzip.GzipFile(fileobj=r, mode='rb')
        html = gz.read()
        gz.close()
        headers["Content-type"] = "text/html; charset=utf-8"
        r.set_data( html )
        b.set_response(r)

    
def login(url):
    br = getBrowserObject()
    br.open(url)
    response = br.open(url)
    ungzipResponse(response, br)
    #html = response.read()
    #print html
    br.select_form(name="signinWidget")
    br.form['username'] = "kshitij.sood@saholic.com"
    br.form['password'] = "pioneer"
    response = br.submit()
    print "********************"
    print "Attempting to Login"
    print "********************"
    #ungzipResponse(response, br)
    return br

        

if __name__ == "__main__":
    parser = optparse.OptionParser()
    parser.add_option("-l", "--logfile", dest="logfile",
                      type="string",
                      help="Log all output to LOG_FILE",
                      )
    parser.add_option("-i", "--pidfile", dest="pidfile",
                      type="string",
                      help="Write the PID to pidfile")
    (options, args) = parser.parse_args()
    daemon = CompetitorScraping(options.logfile, options.pidfile)
    if len(args) == 0:
        daemon.run()
    elif len(args) == 1:
        if 'start' == args[0]:
            daemon.start()
        elif 'stop' == args[0]:
            daemon.stop()
        elif 'restart' == args[0]:
            daemon.restart()
        else:
            print "Unknown command"
            sys.exit(2)
        sys.exit(0)
    else:
        print "usage: %s start|stop|restart" % sys.argv[0]
        sys.exit(2)