Subversion Repositories SmartDukaan

Rev

Rev 14255 | Rev 14325 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14255 Rev 14262
Line 37... Line 37...
37 
            if not toScrapMap.has_key(data['_id']):
 37 
            if not toScrapMap.has_key(data['_id']):
38 
                toScrapMap[data['_id']] = data
 38 
                toScrapMap[data['_id']] = data
39 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':2,'showDeal':1,'totalPoints':{'$gt':0}}))
 39 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':2,'showDeal':1,'totalPoints':{'$gt':0}}))
40 
    for deal in dealFlagged:
 40 
    for deal in dealFlagged:
41 
        if not toScrapMap.has_key(deal['_id']):
 41 
        if not toScrapMap.has_key(deal['_id']):
- 
 
 42 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
42 
            toScrapMap[deal['_id']] = deal
 43 
            toScrapMap[deal['_id']] = data[0]
43 
    pool = ThreadPool(cpu_count() *2)
 44 
    pool = ThreadPool(cpu_count() *2)
44 
    pool.map(scrapeFlipkart,toScrapMap.values())
 45 
    pool.map(scrapeFlipkart,toScrapMap.values())
45 
    pool.close()
 46 
    pool.close()
46 
    pool.join()
 47 
    pool.join()
47 
    print "joining threads at %s"%(str(datetime.now()))
 48 
    print "joining threads at %s"%(str(datetime.now()))