Subversion Repositories SmartDukaan

Rev

Rev 14172 | Rev 14251 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14172 Rev 14178
Line 1... Line 1...
1 
import pymongo
 1 
import pymongo
2 
from dtr.utils.utils import to_java_date
 2 
from dtr.utils.utils import to_java_date
3 
from datetime import datetime, timedelta
 3 
from datetime import datetime, timedelta
4 
from operator import itemgetter
 4 
from operator import itemgetter
5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
 5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
6 
from multiprocessing.dummy import Pool as ThreadPool
 6 
from multiprocessing import Pool as ThreadPool
7 
from multiprocessing import cpu_count
 7 
from multiprocessing import cpu_count
8 
import sys
8 
import sys
9 
 
 9 
 
10 
con = None
 10 
con = None
11 
 
 11 
 
Line 30... Line 30...
30 
                toScrapMap[data['_id']] = data
 30 
                toScrapMap[data['_id']] = data
31 
    for k, y in toScrapMap.iteritems():
 31 
    for k, y in toScrapMap.iteritems():
32 
        print k,
 32 
        print k,
33 
        print '\t',
 33 
        print '\t',
34 
        print y
 34 
        print y
35 
    pool = ThreadPool(cpu_count() * 2)
 35 
    pool = ThreadPool(cpu_count() *2)
36 
    pool.map(scrapeFlipkart,toScrapMap.values())
 36 
    pool.map(scrapeFlipkart,toScrapMap.values())
37 
    pool.close()
 37 
    pool.close()
38 
    pool.join()
 38 
    pool.join()
39 
    print "joining threads"
 39 
    print "joining threads"
40 
    print datetime.now()
 40 
    print datetime.now()