Subversion Repositories SmartDukaan

Rev

Rev 14174 | Rev 14256 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14174 Rev 14250
Line 24... Line 24...
24 
    for bestSeller in bestSellers:
24 
    for bestSeller in bestSellers:
25 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
 25 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
26 
        for data in amazonBestSellers:
 26 
        for data in amazonBestSellers:
27 
            if not toScrapMap.has_key(data['_id']):
 27 
            if not toScrapMap.has_key(data['_id']):
28 
                toScrapMap[data['_id']] = data
 28 
                toScrapMap[data['_id']] = data
29 
    for k, y in toScrapMap.iteritems():
 29 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':0}}))
30 
        print k,
 30 
    for deal in dealFlagged:
31 
        print '\t',
 31 
        if not toScrapMap.has_key(deal['_id']):
32 
        print y
 32 
            toScrapMap[deal['_id']] = deal
33 
    pool = ThreadPool(cpu_count() * 2)
 33 
    pool = ThreadPool(cpu_count() * 2)
34 
    pool.map(scrapeAmazon,toScrapMap.values())
 34 
    pool.map(scrapeAmazon,toScrapMap.values())
35 
    pool.close()
 35 
    pool.close()
36 
    pool.join()
 36 
    pool.join()
37 
    print "joining threads"
 37 
    print "joining threads at %s"%(str(datetime.now()))
38 
    print datetime.now()
 - 
 
39
38
40 
 
 39 
 
41 
def scrapeAmazon(data):
 40 
def scrapeAmazon(data):
42 
    inStock = 0
 41 
    inStock = 0
43 
    print str(data['identifier'])
 42 
    print str(data['identifier'])