Subversion Repositories SmartDukaan

Rev

Rev 14127 | Rev 14149 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 14127 Rev 14131
Line 18... Line 18...
18 
            print e
 18 
            print e
19 
            return None
 19 
            return None
20 
    return con
 20 
    return con
21 
 
 21 
 
22 
def scrapeFlipkart():
 22 
def scrapeFlipkart():
23 
    flipkartBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
 23 
    bestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0}}))
24 
    for data in flipkartBestSellers:
 24 
    for bestSeller in bestSellers:
25 
        if data['source_id']!=2:
 - 
 
26 
            continue
 - 
 
27 
        inStock = 0
 - 
 
28 
        retryCount = 0
 - 
 
29 
        print str(data['identifier'])
 - 
 
30 
        if data['identifier'] is None or len(data['identifier'].strip())==0:
 25 
        flipkartBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'skuBundleId':bestSeller['skuBundleId'],'source_id':2}))
31 
            print "continue"
 26 
        for data in flipkartBestSellers:
32 
            continue
 27 
            if data['source_id']!=2:
33
- 
 
34 
        try:
 - 
 
35 
            if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
 - 
 
36 
                print "sku id is already updated",data['_id']
- 
 
37 
                continue
 28 
                continue
38 
        except:
 - 
 
39 
            pass
 - 
 
40
- 
 
41
- 
 
42 
        lowestSp = 0
 - 
 
43 
        inStock = 0
 29 
            inStock = 0
44 
        try:
 - 
 
45 
            if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
 - 
 
46 
                result = scraperProductPage.read(data['marketPlaceUrl'])
 - 
 
47 
                if result.get('lowestSp')!=0:
 - 
 
48 
                    lowestSp = result.get('lowestSp')
 - 
 
49 
                    inStock = result.get('inStock')
 - 
 
50 
        except:
 - 
 
51 
            print "Unable to scrape product page ",data['identifier']
 - 
 
52
- 
 
53
- 
 
54 
        if lowestSp == 0:
 - 
 
55 
            url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
 - 
 
56 
            while(retryCount < 3):
 30 
            retryCount = 0
57 
                try:
 - 
 
58 
                    vendorsData = scraperFk.read(url)
 - 
 
59 
                    fetched = True
 - 
 
60 
                    break
 - 
 
61 
                except Exception as e:
 31 
            print str(data['identifier'])
62 
                    print "***Retry count ",retryCount
32 
            if data['identifier'] is None or len(data['identifier'].strip())==0:
63 
                    retryCount+=1
 - 
 
64 
                    if retryCount == 3:
 - 
 
65 
                        fetched = False
 - 
 
66 
                    print e
 33 
                print "continue"
67 
            if not fetched:
 - 
 
68 
                print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
 - 
 
69 
                continue
 34 
                continue
70
35
71 
            sortedVendorsData = []
 36 
            try:
72 
            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
 37 
                if data['priceUpdatedOn'] > to_java_date(datetime.now() - timedelta(minutes=5)):
73 
            print "data",sortedVendorsData
 38 
                    print "sku id is already updated",data['_id']
74 
            lowestSp, iterator = (0,)*2
 39 
                    continue
75 
            for vData in sortedVendorsData:
 40 
            except:
76 
                if iterator == 0:
 41 
                pass
77 
                    lowestSp = vData['sellingPrice']
 42
78 
                break
 43
79 
            if lowestSp > 0:
 44 
            lowestSp = 0
80 
                inStock = 1
 45 
            inStock = 0
81 
        print lowestSp
 46 
            try:
- 
 
 47 
                if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
82 
        print inStock
 48 
                    result = scraperProductPage.read(data['marketPlaceUrl'])
83 
        if lowestSp > 0:
 49 
                    if result.get('lowestSp')!=0:
84 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
 50 
                        lowestSp = result.get('lowestSp')
85 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
 51 
                        inStock = result.get('inStock')
86 
        else:
 52 
            except:
87 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
 - 
 
88 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
 53 
                print "Unable to scrape product page ",data['identifier']
89
54
- 
 
 55
- 
 
 56 
            if lowestSp == 0:
- 
 
 57 
                url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
- 
 
 58 
                while(retryCount < 3):
- 
 
 59 
                    try:
- 
 
 60 
                        vendorsData = scraperFk.read(url)
- 
 
 61 
                        fetched = True
- 
 
 62 
                        break
- 
 
 63 
                    except Exception as e:
- 
 
 64 
                        print "***Retry count ",retryCount
- 
 
 65 
                        retryCount+=1
- 
 
 66 
                        if retryCount == 3:
- 
 
 67 
                            fetched = False
- 
 
 68 
                        print e
- 
 
 69 
                if not fetched:
- 
 
 70 
                    print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
- 
 
 71 
                    continue
- 
 
 72
- 
 
 73 
                sortedVendorsData = []
- 
 
 74 
                sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
- 
 
 75 
                print "data",sortedVendorsData
- 
 
 76 
                lowestSp, iterator = (0,)*2
- 
 
 77 
                for vData in sortedVendorsData:
- 
 
 78 
                    if iterator == 0:
- 
 
 79 
                        lowestSp = vData['sellingPrice']
- 
 
 80 
                    break
- 
 
 81 
                if lowestSp > 0:
- 
 
 82 
                    inStock = 1
- 
 
 83 
            print lowestSp
- 
 
 84 
            print inStock
- 
 
 85 
            if lowestSp > 0:
- 
 
 86 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(datetime.now()),'priceUpdatedOn':to_java_date(datetime.now()),'in_stock':inStock}}, multi=True)
- 
 
 87 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
- 
 
 88 
            else:
- 
 
 89 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(datetime.now()),'in_stock':inStock,'priceUpdatedOn':to_java_date(datetime.now())}}, multi=True)
- 
 
 90 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
- 
 
 91
90 
        try:
 92 
            try:
91 
            recomputeDeal(data['skuBundleId'])
 93 
                recomputeDeal(data['skuBundleId'])
92 
        except:
 94 
            except:
93 
            print "Unable to compute deal for ",data['skuBundleId']
 95 
                print "Unable to compute deal for ",data['skuBundleId']
94 
 
 96 
 
95 
def recomputeDeal(skuBundleId):
 97 
def recomputeDeal(skuBundleId):
96 
    """Lets recompute deal for this bundle"""
 98 
    """Lets recompute deal for this bundle"""
97 
    print "Recomputing for bundleId",skuBundleId
 99 
    print "Recomputing for bundleId",skuBundleId
98
100