Subversion Repositories SmartDukaan

Rev

Rev 16501 | Rev 16869 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 16501 Rev 16509
Line 42... Line 42...
42 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
 42 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':1}))
43 
        for data in amazonBestSellers:
 43 
        for data in amazonBestSellers:
44 
            if not toScrapMap.has_key(data['_id']):
 44 
            if not toScrapMap.has_key(data['_id']):
45 
                data['dealFlag'] = 0
 45 
                data['dealFlag'] = 0
46 
                data['dealType'] = 0
 46 
                data['dealType'] = 0
- 
 
 47 
                data['dealUrl'] = ""
47 
                toScrapMap[data['_id']] = data
 48 
                toScrapMap[data['_id']] = data
48 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':-100}}))
 49 
    dealFlagged = list(get_mongo_connection().Catalog.Deals.find({'source_id':1,'showDeal':1,'totalPoints':{'$gt':-100}}))
49 
    for deal in dealFlagged:
 50 
    for deal in dealFlagged:
50 
        if not toScrapMap.has_key(deal['_id']):
 51 
        if not toScrapMap.has_key(deal['_id']):
51 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
 52 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':deal['_id']}))
52 
            data[0]['dealFlag'] = 0
 53 
            data[0]['dealFlag'] = 0
53 
            data[0]['dealType'] = 0
 54 
            data[0]['dealType'] = 0
- 
 
 55 
            data[0]['dealUrl'] = ""
54 
            toScrapMap[deal['_id']] = data[0]
 56 
            toScrapMap[deal['_id']] = data[0]
55 
    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':1}))
 57 
    manualDeals = list(get_mongo_connection().Catalog.ManualDeals.find({'startDate':{'$lte':to_java_date(datetime.now())},'endDate':{'$gte':to_java_date(datetime.now())},'source_id':1}))
56 
    for manualDeal in manualDeals:
 58 
    for manualDeal in manualDeals:
57 
        if not toScrapMap.has_key(manualDeal['sku']):
 59 
        if not toScrapMap.has_key(manualDeal['sku']):
58 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
 60 
            data = list(get_mongo_connection().Catalog.MasterData.find({'_id':manualDeal['sku']}))
59 
            if len(data) > 0:
 61 
            if len(data) > 0:
60 
                data[0]['dealFlag'] = 1
 62 
                data[0]['dealFlag'] = 1
61 
                data[0]['dealType'] = manualDeal['dealType']
 63 
                data[0]['dealType'] = manualDeal['dealType']
- 
 
 64 
                data[0]['dealUrl'] = manualDeal['dealUrl']
62 
                toScrapMap[manualDeal['sku']] = data[0]
 65 
                toScrapMap[manualDeal['sku']] = data[0]
63 
        else:
 66 
        else:
64 
            data = toScrapMap.get(manualDeal['sku'])
 67 
            data = toScrapMap.get(manualDeal['sku'])
65 
            data['dealFlag'] = 1
 68 
            data['dealFlag'] = 1
66 
            data['dealType'] = manualDeal['dealType']
 69 
            data['dealType'] = manualDeal['dealType']
- 
 
 70 
            data['dealUrl'] = manualDeal['dealUrl']
67 
    pool = ThreadPool(cpu_count() * 2)
 71 
    pool = ThreadPool(cpu_count() * 2)
68 
    pool.map(scrapeAmazon,toScrapMap.values())
 72 
    pool.map(scrapeAmazon,toScrapMap.values())
69 
    pool.close()
 73 
    pool.close()
70 
    pool.join()
 74 
    pool.join()
71 
    print "joining threads at %s"%(str(datetime.now()))
 75 
    print "joining threads at %s"%(str(datetime.now()))
Line 91... Line 95...
91
95
92 
    url = "http://www.amazon.in/gp/aw/ol/%s?o=New&op=1"%(data['identifier'])
 96 
    url = "http://www.amazon.in/gp/aw/ol/%s?o=New&op=1"%(data['identifier'])
93
97
94 
    try:
 98 
    try:
95 
        if data['dealFlag'] ==1 and data['dealType'] ==1:
 99 
        if data['dealFlag'] ==1 and data['dealType'] ==1:
96 
            deal_url = "http://www.amazon.in/dp/%s"%(data['identifier'].strip())
 100 
            deal_url = data['dealUrl'].strip()
97 
            dealScraping = True
 101 
            dealScraping = True
98 
    except:
 102 
    except:
99 
        data['dealFlag'] = 0
 103 
        data['dealFlag'] = 0
100 
        data['dealType'] = 0
 104 
        data['dealType'] = 0
- 
 
 105 
        data['dealUrl'] = ""
101
106
102 
    print url
 107 
    print url
103 
    lowestPrice = 0.0
 108 
    lowestPrice = 0.0
104
109
105 
    if not dealScraping:
 110 
    if not dealScraping: