Subversion Repositories SmartDukaan

Rev

Rev 12292 | Rev 12314 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 12292 Rev 12297
Line 3... Line 3...
3 
from sqlalchemy.sql import asc
 3 
from sqlalchemy.sql import asc
4 
from sqlalchemy.sql.expression import or_
 4 
from sqlalchemy.sql.expression import or_
5 
from shop2020.utils.daemon import Daemon
 5 
from shop2020.utils.daemon import Daemon
6 
import optparse
 6 
import optparse
7 
import sys
 7 
import sys
8 
import threading
 - 
 
9 
import urllib2
 - 
 
10 
import mechanize
 8 
import mechanize
11 
import cookielib
 - 
 
12 
import time
 9 
import time
13 
import requests as httpRequest
 - 
 
14 
import simplejson as json
 - 
 
15 
from shop2020.model.v1.catalog.impl import DataService
 10 
from shop2020.model.v1.catalog.impl import DataService
16 
from shop2020.model.v1.catalog.script import FlipkartScraper, AmazonScraper, SellerCentralScraper
 - 
 
17 
from operator import itemgetter
 - 
 
18 
from shop2020.model.v1.catalog.impl.DataService import CompetitorPricing, CompetitorPricingRequest, \
 11 
from shop2020.model.v1.catalog.impl.DataService import CompetitorPricing, CompetitorPricingRequest
19 
SnapdealItem, FlipkartItem
 - 
 
20 
import gc
12 
import gc
21 
 
 13 
 
22 
config_client = ConfigClient()
 14 
config_client = ConfigClient()
23 
host = config_client.get_property('staging_hostname')
 15 
host = config_client.get_property('staging_hostname')
24 
DataService.initialize(db_hostname=host)
 16 
DataService.initialize(db_hostname=host)
Line 55... Line 47...
55 
    except Exception as e:
 47 
    except Exception as e:
56 
        print e
 48 
        print e
57 
        sys.exit(2)
 49 
        sys.exit(2)
58
50
59 
def fetchDetails(request):
 51 
def fetchDetails(request):
- 
 
 52 
    import threading
60 
    login_url = "https://sellercentral.amazon.in/gp/homepage.html"
 53 
    login_url = "https://sellercentral.amazon.in/gp/homepage.html"
61 
    br = login(login_url)
 54 
    br = login(login_url)
62 
    items = session.query(CompetitorPricing).filter(CompetitorPricing.competitorPricing_requestId==request.requestId).all()
 55 
    items = session.query(CompetitorPricing).filter(CompetitorPricing.competitorPricing_requestId==request.requestId).all()
63 
    print items
 56 
    print items
64 
    snapdeal, flipkart, amazon =[],[],[]
 57 
    snapdeal, flipkart, amazon =[],[],[]
Line 86... Line 79...
86 
        th.join()
 79 
        th.join()
87 
    br,t1,t2,t3 =None,None,None,None
 80 
    br,t1,t2,t3 =None,None,None,None
88 
    items[:],snapdeal[:],flipkart[:],amazon[:],threads[:]=[],[],[],[],[]
 81 
    items[:],snapdeal[:],flipkart[:],amazon[:],threads[:]=[],[],[],[],[]
89 
 
 82 
 
90 
def scrapSnapdeal(snapdealItems):
 83 
def scrapSnapdeal(snapdealItems):
- 
 
 84 
    import simplejson as json
- 
 
 85 
    import urllib2
- 
 
 86 
    from shop2020.model.v1.catalog.impl.DataService import SnapdealItem
91 
    for snapdealItem in snapdealItems:
 87 
    for snapdealItem in snapdealItems:
92 
        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)
 88 
        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)
93 
        if sdItem is None:
 89 
        if sdItem is None:
94 
            continue
 90 
            continue
95 
        try:
 91 
        try:
Line 131... Line 127...
131 
        snapdealItem.lowestSnapdealOfferPrice = lowestOfferPrice
 127 
        snapdealItem.lowestSnapdealOfferPrice = lowestOfferPrice
132 
        snapdealItem.lowestSnapdealSeller = lowestSellerName
128 
        snapdealItem.lowestSnapdealSeller = lowestSellerName
133 
        snapdealItem.lowestSnapdealSellerInventory = lowestSellerInventory
129 
        snapdealItem.lowestSnapdealSellerInventory = lowestSellerInventory
134 
 
 130 
 
135 
def scrapFlipkart(flipkartItems):
 131 
def scrapFlipkart(flipkartItems):
- 
 
 132 
    from shop2020.model.v1.catalog.script import FlipkartScraper
- 
 
 133 
    from operator import itemgetter
- 
 
 134 
    import requests as httpRequest
- 
 
 135 
    from shop2020.model.v1.catalog.impl.DataService import FlipkartItem
136 
    scraperFk = FlipkartScraper.FlipkartScraper()
 136 
    scraperFk = FlipkartScraper.FlipkartScraper()
137 
    for flipkartItem in flipkartItems:
 137 
    for flipkartItem in flipkartItems:
138 
        fkItem = FlipkartItem.get_by(item_id=flipkartItem.item_id)
 138 
        fkItem = FlipkartItem.get_by(item_id=flipkartItem.item_id)
139 
        if fkItem is None:
 139 
        if fkItem is None:
140 
            continue
 140 
            continue
Line 178... Line 178...
178 
    if session.is_active:
 178 
    if session.is_active:
179 
        print "session is active. closing it."
 179 
        print "session is active. closing it."
180 
        session.close()
 180 
        session.close()
181
181
182 
def scrapAmazon(amazonItems,br):
 182 
def scrapAmazon(amazonItems,br):
- 
 
 183 
    from shop2020.model.v1.catalog.script import AmazonScraper, SellerCentralScraper
183 
    print "Inside amazonitems ",amazonItems
 184 
    print "Inside amazonitems ",amazonItems
184 
    print "len amazon items ",len(amazonItems)
 185 
    print "len amazon items ",len(amazonItems)
185 
    time.sleep(5)
 186 
    time.sleep(5)
186 
    sc = SellerCentralScraper.SellerCentralScraper()
 187 
    sc = SellerCentralScraper.SellerCentralScraper()
187 
    scraperAmazon = AmazonScraper.AmazonScraper()
 188 
    scraperAmazon = AmazonScraper.AmazonScraper()
Line 218... Line 219...
218 
            continue
 219 
            continue
219 
    sc =None
 220 
    sc =None
220 
    scraperAmazon = None
 221 
    scraperAmazon = None
221
222
222 
def getBrowserObject():
 223 
def getBrowserObject():
- 
 
 224 
        import cookielib
223 
        br = mechanize.Browser(factory=mechanize.RobustFactory())
 225 
        br = mechanize.Browser(factory=mechanize.RobustFactory())
224 
        cj = cookielib.LWPCookieJar()
 226 
        cj = cookielib.LWPCookieJar()
225 
        br.set_cookiejar(cj)
 227 
        br.set_cookiejar(cj)
226 
        br.set_handle_equiv(True)
 228 
        br.set_handle_equiv(True)
227 
        br.set_handle_redirect(True)
 229 
        br.set_handle_redirect(True)