Subversion Repositories SmartDukaan

Rev

Rev 14149 | Rev 14158 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14149 Rev 14157
Line 4... Line 4...
4 
from operator import itemgetter
 4 
from operator import itemgetter
5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
 5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
6 
from multiprocessing.dummy import Pool as ThreadPool
6 
from multiprocessing.dummy import Pool as ThreadPool
7 
 
 7 
 
8 
con = None
 8 
con = None
9 
scraperFk = FlipkartScraper.FlipkartScraper()
 - 
 
10 
scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
 - 
 
11 
 
 9 
 
12 
def get_mongo_connection(host='localhost', port=27017):
 10 
def get_mongo_connection(host='localhost', port=27017):
13 
    global con
 11 
    global con
14 
    if con is None:
 12 
    if con is None:
15 
        print "Establishing connection %s host and port %d" %(host,port)
 13 
        print "Establishing connection %s host and port %d" %(host,port)
Line 38... Line 36...
38 
    pool.join()
 36 
    pool.join()
39 
    print "joining threads"
 37 
    print "joining threads"
40 
 
 38 
 
41 
def scrapeFlipkart(data):
 39 
def scrapeFlipkart(data):
42 
    if data['source_id']!=2:
 40 
    if data['source_id']!=2:
43 
        continue
 41 
        return
44 
    inStock = 0
 42 
    inStock = 0
45 
    retryCount = 0
 43 
    retryCount = 0
46 
    print str(data['identifier'])
 44 
    print str(data['identifier'])
47 
    if data['identifier'] is None or len(data['identifier'].strip())==0:
 45 
    if data['identifier'] is None or len(data['identifier'].strip())==0:
48 
        print "continue"
 46 
        print "returning in valid identifier"
49 
        continue
 47 
        return
50
48
51 
    try:
 49 
    try:
52 
        if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
 50 
        if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
53 
            print "sku id is already updated",data['_id']
51 
            print "sku id is already updated",data['_id']
54 
            continue
 52 
            return
55 
    except:
 53 
    except:
56 
        pass
 54 
        pass
57
55
58
56
59 
    lowestSp = 0
 57 
    lowestSp = 0
60 
    inStock = 0
 58 
    inStock = 0
- 
 
 59 
    scraperFk = FlipkartScraper.FlipkartScraper()
- 
 
 60 
    scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
61 
    try:
 61 
    try:
62 
        if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
 62 
        if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
63 
            result = scraperProductPage.read(data['marketPlaceUrl'])
 63 
            result = scraperProductPage.read(data['marketPlaceUrl'])
64 
            if result.get('lowestSp')!=0:
 64 
            if result.get('lowestSp')!=0:
65 
                lowestSp = result.get('lowestSp')
 65 
                lowestSp = result.get('lowestSp')
Line 81... Line 81...
81 
                if retryCount == 3:
 81 
                if retryCount == 3:
82 
                    fetched = False
 82 
                    fetched = False
83 
                print e
 83 
                print e
84 
        if not fetched:
 84 
        if not fetched:
85 
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
 85 
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
86 
            continue
 86 
            return
87
87
88 
        sortedVendorsData = []
 88 
        sortedVendorsData = []
89 
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
 89 
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
90 
        print "data",sortedVendorsData
 90 
        print "data",sortedVendorsData
91 
        lowestSp, iterator = (0,)*2
 91 
        lowestSp, iterator = (0,)*2
Line 139... Line 139...
139 
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
 139 
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
140 
    if len(toUpdate) > 0:
 140 
    if len(toUpdate) > 0:
141 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
 141 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
142 
 
 142 
 
143 
def main():
 143 
def main():
144 
    scrapeFlipkart()
 144 
    populate()
145
145
146 
if __name__=='__main__':
 146 
if __name__=='__main__':
147 
    main()
 147 
    main()
148 

            148