| Line 2164... |
Line 2164... |
| 2164 |
count = 0
|
2164 |
count = 0
|
| 2165 |
pool = ThreadPool(3)
|
2165 |
pool = ThreadPool(3)
|
| 2166 |
startOffset = 0
|
2166 |
startOffset = 0
|
| 2167 |
endOffset = startOffset
|
2167 |
endOffset = startOffset
|
| 2168 |
while(count<3 and endOffset<len(itemInfo)):
|
2168 |
while(count<3 and endOffset<len(itemInfo)):
|
| 2169 |
endOffset = startOffset + 5
|
2169 |
endOffset = startOffset + 20
|
| 2170 |
if (endOffset >= len(itemInfo)):
|
2170 |
if (endOffset >= len(itemInfo)):
|
| 2171 |
endOffset = len(itemInfo)
|
2171 |
endOffset = len(itemInfo)
|
| 2172 |
print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
|
2172 |
print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
|
| 2173 |
pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
|
2173 |
pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
|
| 2174 |
#t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
|
2174 |
#t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
|
| 2175 |
#t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
|
2175 |
#t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
|
| 2176 |
#t = threading.Thread(target=partial(test, startOffset, endOffset))
|
2176 |
#t = threading.Thread(target=partial(test, startOffset, endOffset))
|
| 2177 |
#threads.append(t)
|
2177 |
#threads.append(t)
|
| 2178 |
startOffset = startOffset + 5
|
2178 |
startOffset = startOffset + 20
|
| 2179 |
count+=1
|
2179 |
count+=1
|
| 2180 |
#[t.start() for t in threads]
|
2180 |
#[t.start() for t in threads]
|
| 2181 |
#[t.join() for t in threads]
|
2181 |
#[t.join() for t in threads]
|
| 2182 |
#threads = []
|
2182 |
#threads = []
|
| 2183 |
pool.close()
|
2183 |
pool.close()
|
| 2184 |
pool.join()
|
2184 |
pool.join()
|
| 2185 |
return endOffset
|
2185 |
return endOffset
|
| 2186 |
else:
|
2186 |
else:
|
| 2187 |
count = 0
|
2187 |
count = 0
|
| 2188 |
pool = ThreadPool(20)
|
2188 |
pool = ThreadPool(50)
|
| 2189 |
startOffset = 0
|
2189 |
startOffset = 0
|
| 2190 |
endOffset = startOffset
|
2190 |
endOffset = startOffset
|
| 2191 |
while(count<1 and endOffset<len(itemInfo)):
|
2191 |
while(count<1 and endOffset<len(itemInfo)):
|
| 2192 |
endOffset = startOffset + 20
|
2192 |
endOffset = startOffset + 50
|
| 2193 |
if (endOffset >= len(itemInfo)):
|
2193 |
if (endOffset >= len(itemInfo)):
|
| 2194 |
endOffset = len(itemInfo)
|
2194 |
endOffset = len(itemInfo)
|
| 2195 |
print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
|
2195 |
print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
|
| 2196 |
pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
|
2196 |
pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
|
| 2197 |
#t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
|
2197 |
#t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
|
| 2198 |
#t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
|
2198 |
#t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
|
| 2199 |
#t = threading.Thread(target=partial(test, startOffset, endOffset))
|
2199 |
#t = threading.Thread(target=partial(test, startOffset, endOffset))
|
| 2200 |
#threads.append(t)
|
2200 |
#threads.append(t)
|
| 2201 |
startOffset = startOffset + 20
|
2201 |
startOffset = startOffset + 50
|
| 2202 |
count+=1
|
2202 |
count+=1
|
| 2203 |
#[t.start() for t in threads]
|
2203 |
#[t.start() for t in threads]
|
| 2204 |
#[t.join() for t in threads]
|
2204 |
#[t.join() for t in threads]
|
| 2205 |
#threads = []
|
2205 |
#threads = []
|
| 2206 |
print "terminating while"
|
2206 |
print "terminating while"
|