Subversion Repositories SmartDukaan

Rev

Rev 13977 | Rev 14124 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 13977 Rev 14121
Line 3... Line 3...
3 
import pymongo
 3 
import pymongo
4 
from dtr.utils.utils import to_java_date
 4 
from dtr.utils.utils import to_java_date
5 
from datetime import datetime, timedelta
 5 
from datetime import datetime, timedelta
6 
from operator import itemgetter
 6 
from operator import itemgetter
7 
from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper
 7 
from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper
8 
from dtr.utils import FlipkartScraper
 8 
from dtr.utils import FlipkartScraper,NewFlipkartScraper
9 
 
 9 
 
10 
con = None
 10 
con = None
11 
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3}
 11 
SOURCE_MAP = {'AMAZON':1,'FLIPKART':2,'SNAPDEAL':3}
12 
scraperFk = FlipkartScraper.FlipkartScraper()
 12 
scraperFk = FlipkartScraper.FlipkartScraper()
13 
scraperAmazon = AmazonScraper()
 13 
scraperAmazon = AmazonScraper()
- 
 
 14 
scraperProductPage = NewFlipkartScraper.FlipkartProductPageScraper()
14 
 
 15 
 
15 
headers = {
16 
headers = {
16 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
 17 
           'User-agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
17 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
18 
            'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
18 
            'Accept-Language' : 'en-US,en;q=0.8',
19 
            'Accept-Language' : 'en-US,en;q=0.8',
Line 129... Line 130...
129 
                    print "sku id is already updated",data['_id']
 130 
                    print "sku id is already updated",data['_id']
130 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
131 
                    return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':2,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail']}
131 
            except:
 132 
            except:
132 
                pass
 133 
                pass
133
134
134
- 
 
135 
            url = "http://www.flipkart.com/ps/%s"%(data['identifier'])
 - 
 
136 
            vendorsData = scraperFk.read(url)
 - 
 
137 
            sortedVendorsData = []
 - 
 
138 
            sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
 - 
 
139 
            print "data",sortedVendorsData
 - 
 
140 
            lowestSp = 0
 135 
            lowestSp = 0
141 
            inStock = 0
 136 
            inStock = 0
- 
 
 137 
            try:
- 
 
 138 
                if data['marketPlaceUrl']!="" or data['marketPlaceUrl'] !="http://www.flipkart.com/ps/%s"%(data['identifier']):
- 
 
 139 
                    result = scraperProductPage.read(data['marketPlaceUrl'])
- 
 
 140 
                    if result.get('lowestPrice')!=0:
- 
 
 141 
                        lowestSp = result.get('lowestPrice')
- 
 
 142 
                        inStock = result.get('inStock')
- 
 
 143 
            except:
- 
 
 144 
                print "Unable to scrape product page ",data['identifier']
- 
 
 145 
            if lowestSp ==0:
- 
 
 146 
                url = "http://www.flipkart.com/ps/%s"%(data['identifier'])
- 
 
 147 
                vendorsData = scraperFk.read(url)
- 
 
 148 
                sortedVendorsData = []
- 
 
 149 
                sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
- 
 
 150 
                print "data",sortedVendorsData
142 
            for vData in sortedVendorsData:
 151 
                for vData in sortedVendorsData:
143 
                lowestSp = vData['sellingPrice']
 152 
                    lowestSp = vData['sellingPrice']
144 
                break
 153 
                    break
145 
            if lowestSp > 0:
 154 
                if lowestSp > 0:
146 
                inStock = 1
 155 
                    inStock = 1
147 
            print lowestSp
 156 
            print lowestSp
148 
            print inStock
 157 
            print inStock
149 
            if lowestSp > 0:
 158 
            if lowestSp > 0:
150 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
 159 
                get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'priceUpdatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
151 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
 160 
                get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)