Subversion Repositories SmartDukaan

Rev

Rev 14178 | Rev 14255 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14178 Rev 14251
Line 26... Line 26...
26 
    for bestSeller in bestSellers:
26 
    for bestSeller in bestSellers:
27 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
 27 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
28 
        for data in amazonBestSellers:
 28 
        for data in amazonBestSellers:
29 
            if not toScrapMap.has_key(data['_id']):
 29 
            if not toScrapMap.has_key(data['_id']):
30 
                toScrapMap[data['_id']] = data
 30 
                toScrapMap[data['_id']] = data
31 
    for k, y in toScrapMap.iteritems():
 31 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':2,'showDeal':1,'totalPoints':{'$gt':0}}))
32 
        print k,
 32 
    for deal in dealFlagged:
33 
        print '\t',
 33 
        if not toScrapMap.has_key(deal['_id']):
34 
        print y
 34 
            toScrapMap[deal['_id']] = deal
35 
    pool = ThreadPool(cpu_count() *2)
 35 
    pool = ThreadPool(cpu_count() *2)
36 
    pool.map(scrapeFlipkart,toScrapMap.values())
 36 
    pool.map(scrapeFlipkart,toScrapMap.values())
37 
    pool.close()
 37 
    pool.close()
38 
    pool.join()
 38 
    pool.join()
39 
    print "joining threads"
 39 
    print "joining threads at %s"%(str(datetime.now()))
40 
    print datetime.now()
 - 
 
41 
 
 40 
 
42 
def scrapeFlipkart(data):
 41 
def scrapeFlipkart(data):
43 
    if data['source_id']!=2:
 42 
    if data['source_id']!=2:
44 
        return
 43 
        return
45 
    inStock = 0
 - 
 
46 
    retryCount = 0
 44 
    retryCount = 0
47 
    print str(data['identifier'])
 45 
    print str(data['identifier'])
48 
    if data['identifier'] is None or len(data['identifier'].strip())==0:
 46 
    if data['identifier'] is None or len(data['identifier'].strip())==0:
49 
        print "returning in valid identifier"
 47 
        print "returning in valid identifier"
50 
        return
 48 
        return