Subversion Repositories SmartDukaan

Rev

Rev 14256 | Rev 14308 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14256 Rev 14260
Line 36... Line 36...
36 
            if not toScrapMap.has_key(data['_id']):
 36 
            if not toScrapMap.has_key(data['_id']):
37 
                toScrapMap[data['_id']] = data
 37 
                toScrapMap[data['_id']] = data
38 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':0}}))
 38 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':0}}))
39 
    for deal in dealFlagged:
 39 
    for deal in dealFlagged:
40 
        if not toScrapMap.has_key(deal['_id']):
 40 
        if not toScrapMap.has_key(deal['_id']):
- 
 
 41 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
41 
            toScrapMap[deal['_id']] = deal
 42 
            toScrapMap[deal['_id']] = data[0]
42 
    pool = ThreadPool(cpu_count() * 2)
 43 
    pool = ThreadPool(cpu_count() * 2)
43 
    pool.map(scrapeAmazon,toScrapMap.values())
 44 
    pool.map(scrapeAmazon,toScrapMap.values())
44 
    pool.close()
 45 
    pool.close()
45 
    pool.join()
 46 
    pool.join()
46 
    print "joining threads at %s"%(str(datetime.now()))
 47 
    print "joining threads at %s"%(str(datetime.now()))