Subversion Repositories SmartDukaan

Rev

Rev 13979 | Rev 14126 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 13979 Rev 14123
Line 1... Line 1...
1 
import pymongo
 1 
import pymongo
2 
from dtr.utils.utils import to_java_date
 2 
from dtr.utils.utils import to_java_date
3 
from datetime import datetime, timedelta
 3 
from datetime import datetime, timedelta
4 
from operator import itemgetter
 4 
from operator import itemgetter
5 
from dtr.utils import FlipkartScraper
 5 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
6 
 
 6 
 
7 
con = None
 7 
con = None
8 
now = datetime.now()
 8 
now = datetime.now()
9 
print to_java_date(now)
 9 
print to_java_date(now)
10 
scraperFk = FlipkartScraper.FlipkartScraper()
 10 
scraperFk = FlipkartScraper.FlipkartScraper()
- 
 
 11 
scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
11 
 
 12 
 
12 
def get_mongo_connection(host='localhost', port=27017):
 13 
def get_mongo_connection(host='localhost', port=27017):
13 
    global con
 14 
    global con
14 
    if con is None:
 15 
    if con is None:
15 
        print "Establishing connection %s host and port %d" %(host,port)
 16 
        print "Establishing connection %s host and port %d" %(host,port)
Line 35... Line 36...
35 
                print "sku id is already updated",data['_id']
36 
                print "sku id is already updated",data['_id']
36 
                continue
 37 
                continue
37 
        except:
 38 
        except:
38 
            pass
 39 
            pass
39
40
40 
        url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
 - 
 
41 
        while(retryCount < 3):
 - 
 
42 
            try:
 - 
 
43 
                vendorsData = scraperFk.read(url)
 - 
 
44 
                fetched = True
 - 
 
45 
                break
 - 
 
46 
            except Exception as e:
 - 
 
47 
                print "***Retry count ",retryCount
- 
 
48 
                retryCount+=1
 - 
 
49 
                if retryCount == 3:
 - 
 
50 
                    fetched = False
 - 
 
51 
                print e
 - 
 
52 
        if not fetched:
 - 
 
53 
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
 - 
 
54 
            continue
 - 
 
55
41
- 
 
 42 
        lowestSp = 0
- 
 
 43 
        inStock = 0
- 
 
 44 
        try:
- 
 
 45 
            if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
- 
 
 46 
                result = scraperProductPage.read(data['marketPlaceUrl'])
- 
 
 47 
                if result.get('lowestPrice')!=0:
- 
 
 48 
                    lowestSp = result.get('lowestPrice')
- 
 
 49 
                    inStock = result.get('inStock')
- 
 
 50 
        except:
- 
 
 51 
            print "Unable to scrape product page ",data['identifier']
- 
 
 52
- 
 
 53
- 
 
 54 
        if lowestSp == 0:
- 
 
 55 
            url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
- 
 
 56 
            while(retryCount < 3):
- 
 
 57 
                try:
- 
 
 58 
                    vendorsData = scraperFk.read(url)
- 
 
 59 
                    fetched = True
- 
 
 60 
                    break
- 
 
 61 
                except Exception as e:
- 
 
 62 
                    print "***Retry count ",retryCount
- 
 
 63 
                    retryCount+=1
- 
 
 64 
                    if retryCount == 3:
- 
 
 65 
                        fetched = False
- 
 
 66 
                    print e
- 
 
 67 
            if not fetched:
- 
 
 68 
                print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
- 
 
 69 
                continue
- 
 
 70
56 
        sortedVendorsData = []
 71 
            sortedVendorsData = []
57 
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
 72 
            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
58 
        print "data",sortedVendorsData
 73 
            print "data",sortedVendorsData
59 
        lowestSp, iterator = (0,)*2
 74 
            lowestSp, iterator = (0,)*2
60 
        for vData in sortedVendorsData:
 75 
            for vData in sortedVendorsData:
61 
            if iterator == 0:
 76 
                if iterator == 0:
62 
                lowestSp = vData['sellingPrice']
 77 
                    lowestSp = vData['sellingPrice']
63 
            break
 78 
                break
64 
        if lowestSp > 0:
 79 
            if lowestSp > 0:
65 
            inStock = 1
 80 
                inStock = 1
66 
        print lowestSp
 81 
        print lowestSp
67 
        print inStock
 82 
        print inStock
68 
        if lowestSp > 0:
 83 
        if lowestSp > 0:
69 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
 84 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
70 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
 85 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)