Subversion Repositories SmartDukaan

Rev

Rev 11560 | Rev 11562 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 11560 Rev 11561
Line 30... Line 30...
30 
from email.mime.text import MIMEText
 30 
from email.mime.text import MIMEText
31 
import email
 31 
import email
32 
from email.mime.multipart import MIMEMultipart
 32 
from email.mime.multipart import MIMEMultipart
33 
import email.encoders
 33 
import email.encoders
34 
import cookielib
 34 
import cookielib
- 
 
 35 
from multiprocessing import Pool
- 
 
 36 
from multiprocessing.dummy import Pool as ThreadPool
35 
 
 37 
 
36 
config_client = ConfigClient()
 38 
config_client = ConfigClient()
37 
host = config_client.get_property('staging_hostname')
 39 
host = config_client.get_property('staging_hostname')
38 
syncPrice=config_client.get_property('sync_price_on_marketplace')
 40 
syncPrice=config_client.get_property('sync_price_on_marketplace')
39 
 
 41 
 
Line 1911... Line 1913...
1911 
        [t.start() for t in threads]
 1913 
        [t.start() for t in threads]
1912 
        [t.join() for t in threads]
1914 
        [t.join() for t in threads]
1913 
        threads = []
 1915 
        threads = []
1914 
        return endOffset
 1916 
        return endOffset
1915 
    else:
 1917 
    else:
- 
 
 1918 
        count = 0
- 
 
 1919 
        pool = ThreadPool(5)
1916 
        startOffset = itemPopulated
 1920 
        startOffset = itemPopulated
1917 
        endOffset = startOffset
 1921 
        endOffset = startOffset
1918 
        while(len(threads)< 10 and endOffset<len(itemInfo)):
 1922 
        while(count<10 and endOffset<len(itemInfo)):
1919 
            endOffset = startOffset + 50
 1923 
            endOffset = startOffset + 50
1920 
            if (endOffset >= len(itemInfo)):
 1924 
            if (endOffset >= len(itemInfo)):
1921 
                endOffset = len(itemInfo)
 1925 
                endOffset = len(itemInfo)
- 
 
 1926 
            pool.map(decideCategory,itemInfo[startOffset:endOffset],scraper)
1922 
            t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
 1927 
            #t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
1923 
            #t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
 1928 
            #t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
1924 
            #t = threading.Thread(target=partial(test, startOffset, endOffset))
 1929 
            #t = threading.Thread(target=partial(test, startOffset, endOffset))
1925 
            threads.append(t)
 1930 
            #threads.append(t)
1926 
            startOffset = startOffset + 50
 1931 
            startOffset = startOffset + 50
- 
 
 1932 
            count+=1
1927 
        [t.start() for t in threads]
 1933 
        #[t.start() for t in threads]
1928 
        [t.join() for t in threads]
1934 
        #[t.join() for t in threads]
1929 
        threads = []
 1935 
        #threads = []
1930 
        return endOffset
 1936 
        return endOffset
1931
1937
1932
1938
1933 
def main():
 1939 
def main():
1934 
    parser = optparse.OptionParser()
 1940 
    parser = optparse.OptionParser()