WebSVN – SmartDukaan – Diff – /trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py

 import email
 from email.mime.multipart import MIMEMultipart
 import email.encoders
 import cookielib
 from multiprocessing import Pool
 from multiprocessing.dummy import Pool as ThreadPool
+from threading import Thread
 config_client = ConfigClient()
 host = config_client.get_property('staging_hostname')
 syncPrice=config_client.get_property('sync_price_on_marketplace')
     else:
         count = 0
         pool = ThreadPool(50)
         startOffset = 0
         endOffset = startOffset
+        threads = []
-        while(count<1 and endOffset<len(itemInfo)):
+        while(count<50 and endOffset<len(itemInfo)):
-            endOffset = startOffset + 50
+            endOffset = startOffset + 1
             if (endOffset >= len(itemInfo)):
                 endOffset = len(itemInfo)
             print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
+            t = Thread(target=populateScrapingResults, args=(itemInfo[startOffset],))
+            t.start()
+            threads.append(t)
-            pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
+            #pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
             #t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
             #t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
             #t = threading.Thread(target=partial(test, startOffset, endOffset))
             #threads.append(t)
-            startOffset = startOffset + 50
+            startOffset = startOffset + 1
             count+=1
         #[t.start() for t in threads]
         #[t.join() for t in threads]
         #threads = []
         print "terminating while"
-        pool.close()
+#        pool.close()
-        pool.join()
+#        pool.join()
+        for thread in threads:
+            thread.join()
         print "joining threads"
         print "returning offset******"
         return endOffset
 def sendAutoPricingMail(successfulAutoDecrease,successfulAutoIncrease):

Subversion Repositories SmartDukaan

(root)/trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py – Rev 12214 → 12216