Subversion Repositories SmartDukaan

Rev

Rev 12271 | Rev 12276 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 12271 Rev 12272
Line 14... Line 14...
14 
import simplejson as json
 14 
import simplejson as json
15 
from shop2020.model.v1.catalog.impl import DataService
 15 
from shop2020.model.v1.catalog.impl import DataService
16 
from shop2020.model.v1.catalog.script import FlipkartScraper, AmazonScraper, SellerCentralScraper
 16 
from shop2020.model.v1.catalog.script import FlipkartScraper, AmazonScraper, SellerCentralScraper
17 
from operator import itemgetter
 17 
from operator import itemgetter
18 
from shop2020.model.v1.catalog.impl.DataService import CompetitorPricing, CompetitorPricingRequest, \
 18 
from shop2020.model.v1.catalog.impl.DataService import CompetitorPricing, CompetitorPricingRequest, \
19 
SnapdealItem, FlipkartItem
19 
SnapdealItem, FlipkartItem
- 
 
 20 
import gc
20 
 
 21 
 
21 
config_client = ConfigClient()
 22 
config_client = ConfigClient()
22 
host = config_client.get_property('staging_hostname')
 23 
host = config_client.get_property('staging_hostname')
23 
DataService.initialize(db_hostname=host)
 24 
DataService.initialize(db_hostname=host)
24 
 
 25 
 
Line 38... Line 39...
38 
                time.sleep(600)
 39 
                time.sleep(600)
39 
            for request in requests:
 40 
            for request in requests:
40 
                fetchDetails(request)
 41 
                fetchDetails(request)
41 
                request.isProcessed = True
 42 
                request.isProcessed = True
42 
                session.commit()
 43 
                session.commit()
- 
 
 44 
                gc.collect()
43 
            close_session()
 45 
            close_session()
44 
    except Exception as e:
 46 
    except Exception as e:
45 
        print e
 47 
        print e
46 
        sys.exit(2)
 48 
        sys.exit(2)
47
49
Line 82... Line 84...
82 
    threads.append(t2)
 84 
    threads.append(t2)
83 
    threads.append(t3)
 85 
    threads.append(t3)
84 
    for th in threads:
 86 
    for th in threads:
85 
        th.join()
 87 
        th.join()
86 
 
 88 
 
- 
 
 89 
 
87 
def scrapSnapdeal(snapdealItems):
 90 
def scrapSnapdeal(snapdealItems):
88 
    for snapdealItem in snapdealItems:
 91 
    for snapdealItem in snapdealItems:
89 
        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)
 92 
        sdItem = SnapdealItem.get_by(item_id=snapdealItem.item_id)
90 
        if sdItem is None:
 93 
        if sdItem is None:
91 
            continue
 94 
            continue