WebSVN – SmartDukaan – Diff – /trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py



        count = 0
        pool = ThreadPool(10)
        startOffset = 0
        endOffset = startOffset
        while(count<10 and endOffset<len(itemInfo)):
            endOffset = startOffset + 50
            if (endOffset >= len(itemInfo)):
                endOffset = len(itemInfo)
            print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
            pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
            #t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
            #t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
            #t = threading.Thread(target=partial(test, startOffset, endOffset))
            #threads.append(t)
            startOffset = startOffset + 50
            count+=1
        #[t.start() for t in threads]
        #[t.join() for t in threads] 
        #threads = []
        print "terminating while"

Rev 12184	Rev 12185
Line 2148...	Line 2148...
2148	`count = 0`	2148	`count = 0`
2149	`pool = ThreadPool(10)`	2149	`pool = ThreadPool(10)`
2150	`startOffset = 0`	2150	`startOffset = 0`
2151	`endOffset = startOffset`	2151	`endOffset = startOffset`
2152	`while(count<10 and endOffset<len(itemInfo)):`	2152	`while(count<10 and endOffset<len(itemInfo)):`
2153	`endOffset = startOffset + 20`	2153	`endOffset = startOffset + 50`
2154	`if (endOffset >= len(itemInfo)):`	2154	`if (endOffset >= len(itemInfo)):`
2155	`endOffset = len(itemInfo)`	2155	`endOffset = len(itemInfo)`
2156	`print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)`	2156	`print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)`
2157	`pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])`	2157	`pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])`
2158	`#t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))`	2158	`#t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))`
2159	`#t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))`	2159	`#t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))`
2160	`#t = threading.Thread(target=partial(test, startOffset, endOffset))`	2160	`#t = threading.Thread(target=partial(test, startOffset, endOffset))`
2161	`#threads.append(t)`	2161	`#threads.append(t)`
2162	`startOffset = startOffset + 20`	2162	`startOffset = startOffset + 50`
2163	`count+=1`	2163	`count+=1`
2164	`#[t.start() for t in threads]`	2164	`#[t.start() for t in threads]`
2165	`#[t.join() for t in threads]`	2165	`#[t.join() for t in threads]`
2166	`#threads = []`	2166	`#threads = []`
2167	`print "terminating while"`	2167	`print "terminating while"`

Subversion Repositories SmartDukaan

(root)/trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py – Rev 12184 → 12185