Subversion Repositories SmartDukaan

Rev

Rev 16407 | Rev 16416 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 16407 Rev 16414
Line 7... Line 7...
7 
from multiprocessing import cpu_count
 7 
from multiprocessing import cpu_count
8 
import optparse
 8 
import optparse
9 
from dtr.storage.MemCache import MemCache
 9 
from dtr.storage.MemCache import MemCache
10 
from dtr.utils.utils import getCashBack
 10 
from dtr.utils.utils import getCashBack
11 
import traceback
 11 
import traceback
- 
 
 12 
from dtr.CouponMaster import addToPaytmMaster
12 
 
 13 
 
13 
con = None
 14 
con = None
14 
 
 15 
 
15 
parser = optparse.OptionParser()
 16 
parser = optparse.OptionParser()
16 
parser.add_option("-m", "--m", dest="mongoHost",
 17 
parser.add_option("-m", "--m", dest="mongoHost",
Line 71... Line 72...
71 
            data = toScrapMap.get(manualDeal['sku'])
 72 
            data = toScrapMap.get(manualDeal['sku'])
72 
            data['dealFlag'] = 1
 73 
            data['dealFlag'] = 1
73 
            data['dealType'] = manualDeal['dealType']
 74 
            data['dealType'] = manualDeal['dealType']
74 
            data['dealPoints'] = manualDeal['dealPoints']
 75 
            data['dealPoints'] = manualDeal['dealPoints']
75 
            data['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
 76 
            data['manualDealThresholdPrice'] = manualDeal['dealThresholdPrice']
76 
    pool = ThreadPool(cpu_count() *2)
 - 
 
77 
    pool.map(scrapePaytm,toScrapMap.values())
 77 
    for val in toScrapMap.values():
78 
    pool.close()
 78 
        scrapePaytm(val)
79 
    pool.join()
 - 
 
80 
    print "joining threads at %s"%(str(datetime.now()))
 79 
    print "joining threads at %s"%(str(datetime.now()))
81 
 
 80 
 
82 
def scrapePaytm(data):
 81 
def scrapePaytm(data):
83 
    if data['source_id']!=6:
 82 
    if data['source_id']!=6:
84 
        return
 83 
        return
85 
    retryCount = 0
 - 
 
86 
    if data['identifier'] is None or len(data['identifier'].strip())==0:
 84 
    if data['identifier'] is None or len(data['identifier'].strip())==0:
87 
        print "returning in valid identifier"
 85 
        print "returning in valid identifier"
88 
        return
 86 
        return
89
87
90 
    if data['_id'] in ignoreItems:
 88 
    if data['_id'] in ignoreItems:
Line 100... Line 98...
100
98
101 
    paytmScraper = PaytmScraper.PaytmScraper()
 99 
    paytmScraper = PaytmScraper.PaytmScraper()
102 
    url = "https://catalog.paytm.com/v1/mobile/product/%s"%(data['identifier'].strip())
 100 
    url = "https://catalog.paytm.com/v1/mobile/product/%s"%(data['identifier'].strip())
103 
    try:
 101 
    try:
104 
        result = paytmScraper.read(url)
 102 
        result = paytmScraper.read(url)
- 
 
 103 
        print result
105 
        effective_price = result.get('offerPrice')
 104 
        effective_price = result.get('offerPrice')
106 
        coupon = ""
 105 
        coupon = ""
- 
 
 106 
        print result['offerUrl']
107 
        try:
 107 
        try:
108 
            offers = PaytmOfferScraper.fetchOffers(result['offerUrl'])
 108 
            offers = PaytmOfferScraper.fetchOffers(result['offerUrl'])
- 
 
 109 
            try:
- 
 
 110 
                addToPaytmMaster(offers.get('codes'))
- 
 
 111 
            except:
- 
 
 112 
                print "Error in adding coupon"
- 
 
 113 
                traceback.print_exc()
109 
            bestOffer = {}
 114 
            bestOffer = {}
110 
            for offer_data in offers.get('codes'):
 115 
            for offer_data in offers.get('codes'):
111 
                if effective_price < offer_data.get('effective_price'):
 116 
                if effective_price > offer_data.get('effective_price'):
112 
                    effective_price = offer_data.get('effective_price')
 117 
                    effective_price = offer_data.get('effective_price')
113 
                    bestOffer = offer_data
 118 
                    bestOffer = offer_data
114 
                    coupon = bestOffer.get('code')
 119 
                    coupon = bestOffer.get('code')
115 
        except:
 120 
        except:
116 
            pass
 121 
            traceback.print_exc()
- 
 
 122 
        print "coupon code",coupon
117 
        available_price = effective_price
123 
        available_price = effective_price
118 
        if result['inStock']:
 124 
        if result['inStock']:
119 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':available_price,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':1,'buyBoxFlag':1,'codAvailable':result.get('cod'),'coupon':coupon}})
 125 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':available_price,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':1,'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon}})
120 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':available_price , 'in_stock':1,'codAvailable':result.get('cod')}})
 126 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':available_price , 'in_stock':1,'codAvailable':result.get('codAvailable')}})
121 
        else:
 127 
        else:
122 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1,'codAvailable':result.get('cod'),'coupon':coupon}})
 128 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1,'codAvailable':result.get('codAvailable'),'coupon':coupon}})
123 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'codAvailable':result.get('cod')}})
 129 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0,'codAvailable':result.get('codAvailable')}})
124
130
125 
    except:
 131 
    except:
126 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1}})
 132 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':0,'priceUpdatedOn':to_java_date(datetime.now()),'buyBoxFlag':1}})
127 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})
 133 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':0}})
128
134