Subversion Repositories SmartDukaan

Rev

Rev 14131 | Rev 14157 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14131 Rev 14149
Line 1... Line 1...
1 
import pymongo
 1 
import pymongo
2 
from dtr.utils.utils import to_java_date
 2 
from dtr.utils.utils import to_java_date
3 
from datetime import datetime, timedelta
 3 
from datetime import datetime, timedelta
4 
from operator import itemgetter
 4 
from operator import itemgetter
5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
 5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
- 
 
 6 
from multiprocessing.dummy import Pool as ThreadPool
6 
 
 7 
 
7 
con = None
 8 
con = None
8 
scraperFk = FlipkartScraper.FlipkartScraper()
 9 
scraperFk = FlipkartScraper.FlipkartScraper()
9 
scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
 10 
scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
10 
 
 11 
 
Line 17... Line 18...
17 
        except Exception, e:
 18 
        except Exception, e:
18 
            print e
 19 
            print e
19 
            return None
 20 
            return None
20 
    return con
 21 
    return con
21 
 
 22 
 
22 
def scrapeFlipkart():
 23 
def populate():
- 
 
 24 
    toScrapMap = {}
23 
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
 25 
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
24 
    for bestSeller in bestSellers:
26 
    for bestSeller in bestSellers:
25 
        flipkartBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
 27 
        amazonBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
26 
        for data in flipkartBestSellers:
 28 
        for data in amazonBestSellers:
- 
 
 29 
            if not toScrapMap.has_key(data['_id']):
- 
 
 30 
                toScrapMap[data['_id']] = data
- 
 
 31 
    for k, y in toScrapMap.iteritems():
- 
 
 32 
        print k,
- 
 
 33 
        print '\t',
- 
 
 34 
        print y
- 
 
 35 
    pool = ThreadPool(50)
- 
 
 36 
    pool.map(scrapeFlipkart,toScrapMap.values())
- 
 
 37 
    pool.close()
- 
 
 38 
    pool.join()
- 
 
 39 
    print "joining threads"
- 
 
 40 
 
- 
 
 41 
def scrapeFlipkart(data):
27 
            if data['source_id']!=2:
 42 
    if data['source_id']!=2:
28 
                continue
 43 
        continue
29 
            inStock = 0
 44 
    inStock = 0
30 
            retryCount = 0
 45 
    retryCount = 0
31 
            print str(data['identifier'])
 46 
    print str(data['identifier'])
32 
            if data['identifier'] is None or len(data['identifier'].strip())==0:
 47 
    if data['identifier'] is None or len(data['identifier'].strip())==0:
33 
                print "continue"
 48 
        print "continue"
34 
                continue
 49 
        continue
35
50
36 
            try:
 51 
    try:
37 
                if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
 52 
        if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
38 
                    print "sku id is already updated",data['_id']
53 
            print "sku id is already updated",data['_id']
39 
                    continue
 54 
            continue
40 
            except:
 55 
    except:
41 
                pass
 56 
        pass
42
57
43
58
44 
            lowestSp = 0
 59 
    lowestSp = 0
45 
            inStock = 0
 60 
    inStock = 0
46 
            try:
 61 
    try:
47 
                if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
 62 
        if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
48 
                    result = scraperProductPage.read(data['marketPlaceUrl'])
 63 
            result = scraperProductPage.read(data['marketPlaceUrl'])
49 
                    if result.get('lowestSp')!=0:
 64 
            if result.get('lowestSp')!=0:
50 
                        lowestSp = result.get('lowestSp')
 65 
                lowestSp = result.get('lowestSp')
51 
                        inStock = result.get('inStock')
 66 
                inStock = result.get('inStock')
52 
            except:
 67 
    except:
53 
                print "Unable to scrape product page ",data['identifier']
 68 
        print "Unable to scrape product page ",data['identifier']
54
69
55
70
56 
            if lowestSp == 0:
 71 
    if lowestSp == 0:
57 
                url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
 72 
        url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
58 
                while(retryCount < 3):
 73 
        while(retryCount < 3):
59 
                    try:
 - 
 
60 
                        vendorsData = scraperFk.read(url)
 - 
 
61 
                        fetched = True
 - 
 
62 
                        break
 - 
 
63 
                    except Exception as e:
 - 
 
64 
                        print "***Retry count ",retryCount
- 
 
65 
                        retryCount+=1
 - 
 
66 
                        if retryCount == 3:
 - 
 
67 
                            fetched = False
 - 
 
68 
                        print e
 - 
 
69 
                if not fetched:
 - 
 
70 
                    print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
 - 
 
71 
                    continue
 - 
 
72
- 
 
73 
                sortedVendorsData = []
 - 
 
74 
                sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
 - 
 
75 
                print "data",sortedVendorsData
 - 
 
76 
                lowestSp, iterator = (0,)*2
 - 
 
77 
                for vData in sortedVendorsData:
 - 
 
78 
                    if iterator == 0:
 - 
 
79 
                        lowestSp = vData['sellingPrice']
 - 
 
80 
                    break
 - 
 
81 
                if lowestSp > 0:
 - 
 
82 
                    inStock = 1
 - 
 
83 
            print lowestSp
 - 
 
84 
            print inStock
 - 
 
85 
            if lowestSp > 0:
 - 
 
86 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
 - 
 
87 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
 - 
 
88 
            else:
 - 
 
89 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
 - 
 
90 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
 - 
 
91
- 
 
92 
            try:
 74 
            try:
- 
 
 75 
                vendorsData = scraperFk.read(url)
- 
 
 76 
                fetched = True
- 
 
 77 
                break
- 
 
 78 
            except Exception as e:
- 
 
 79 
                print "***Retry count ",retryCount
- 
 
 80 
                retryCount+=1
- 
 
 81 
                if retryCount == 3:
- 
 
 82 
                    fetched = False
- 
 
 83 
                print e
- 
 
 84 
        if not fetched:
- 
 
 85 
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
- 
 
 86 
            continue
- 
 
 87
- 
 
 88 
        sortedVendorsData = []
- 
 
 89 
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
- 
 
 90 
        print "data",sortedVendorsData
- 
 
 91 
        lowestSp, iterator = (0,)*2
- 
 
 92 
        for vData in sortedVendorsData:
- 
 
 93 
            if iterator == 0:
- 
 
 94 
                lowestSp = vData['sellingPrice']
- 
 
 95 
            break
- 
 
 96 
        if lowestSp > 0:
- 
 
 97 
            inStock = 1
- 
 
 98 
    print lowestSp
- 
 
 99 
    print inStock
- 
 
 100 
    if lowestSp > 0:
- 
 
 101 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
- 
 
 102 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
- 
 
 103 
    else:
- 
 
 104 
        get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
- 
 
 105 
        get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
- 
 
 106
- 
 
 107 
    try:
93 
                recomputeDeal(data['skuBundleId'])
 108 
        recomputeDeal(data['skuBundleId'])
94 
            except:
 109 
    except:
95 
                print "Unable to compute deal for ",data['skuBundleId']
 110 
        print "Unable to compute deal for ",data['skuBundleId']
96 
 
 111 
 
97 
def recomputeDeal(skuBundleId):
 112 
def recomputeDeal(skuBundleId):
98 
    """Lets recompute deal for this bundle"""
 113 
    """Lets recompute deal for this bundle"""
99 
    print "Recomputing for bundleId",skuBundleId
 114 
    print "Recomputing for bundleId",skuBundleId
100
115