Subversion Repositories SmartDukaan

Rev

Rev 14158 | Rev 14178 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14158 Rev 14172
Line 1... Line 1...
1 
import pymongo
 1 
import pymongo
2 
from dtr.utils.utils import to_java_date
 2 
from dtr.utils.utils import to_java_date
3 
from datetime import datetime, timedelta
 3 
from datetime import datetime, timedelta
4 
from operator import itemgetter
 4 
from operator import itemgetter
5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
 5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
6 
from multiprocessing.dummy import Pool as ThreadPool
6 
from multiprocessing.dummy import Pool as ThreadPool
- 
 
 7 
from multiprocessing import cpu_count
- 
 
 8 
import sys
7 
 
 9 
 
8 
con = None
 10 
con = None
9 
 
 11 
 
10 
def get_mongo_connection(host='localhost', port=27017):
 12 
def get_mongo_connection(host='localhost', port=27017):
11 
    global con
 13 
    global con
Line 28... Line 30...
28 
                toScrapMap[data['_id']] = data
 30 
                toScrapMap[data['_id']] = data
29 
    for k, y in toScrapMap.iteritems():
 31 
    for k, y in toScrapMap.iteritems():
30 
        print k,
 32 
        print k,
31 
        print '\t',
 33 
        print '\t',
32 
        print y
 34 
        print y
33 
    pool = ThreadPool(50)
 35 
    pool = ThreadPool(cpu_count() * 2)
34 
    pool.map(scrapeFlipkart,toScrapMap.values())
 36 
    pool.map(scrapeFlipkart,toScrapMap.values())
35 
    pool.close()
 37 
    pool.close()
36 
    pool.join()
 38 
    pool.join()
37 
    print "joining threads"
 39 
    print "joining threads"
38 
    print datetime.now()
 40 
    print datetime.now()