| Line 2... |
Line 2... |
| 2 |
import simplejson as json
|
2 |
import simplejson as json
|
| 3 |
import pymongo
|
3 |
import pymongo
|
| 4 |
from dtr.utils.utils import to_java_date, getNlcPoints, transformUrl
|
4 |
from dtr.utils.utils import to_java_date, getNlcPoints, transformUrl
|
| 5 |
from datetime import datetime, timedelta
|
5 |
from datetime import datetime, timedelta
|
| 6 |
from operator import itemgetter
|
6 |
from operator import itemgetter
|
| 7 |
from dtr.utils.AmazonPriceOnlyScraper import AmazonScraper
|
7 |
from shop2020.model.v1.catalog.script import AmazonAsyncScraper
|
| 8 |
from dtr.utils import AmazonDealScraper
|
- |
|
| 9 |
from dtr.utils import FlipkartScraper,NewFlipkartScraper, ShopCluesScraper, \
|
8 |
from dtr.utils import FlipkartScraper,NewFlipkartScraper, ShopCluesScraper, \
|
| 10 |
PaytmOfferScraper, PaytmScraper, HomeShop18Scraper
|
9 |
PaytmOfferScraper, PaytmScraper, HomeShop18Scraper
|
| 11 |
from dtr.storage.MemCache import MemCache
|
10 |
from dtr.storage.MemCache import MemCache
|
| 12 |
from functools import partial
|
11 |
from functools import partial
|
| 13 |
import threading
|
12 |
import threading
|
| Line 35... |
Line 34... |
| 35 |
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
34 |
'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
| 36 |
'Accept-Language' : 'en-US,en;q=0.8',
|
35 |
'Accept-Language' : 'en-US,en;q=0.8',
|
| 37 |
'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
|
36 |
'Accept-Charset' : 'ISO-8859-1,utf-8;q=0.7,*;q=0.3'
|
| 38 |
}
|
37 |
}
|
| 39 |
|
38 |
|
| - |
|
39 |
amScraper = AmazonAsyncScraper.Products("AKIAII3SGRXBJDPCHSGQ", "B92xTbNBTYygbGs98w01nFQUhbec1pNCkCsKVfpg", "AF6E3O0VE0X4D")
|
| - |
|
40 |
marketplaceId = 'A21TJRUUN4KGV'
|
| 40 |
|
41 |
|
| 41 |
def get_mongo_connection(port=27017):
|
42 |
def get_mongo_connection(port=27017):
|
| 42 |
global con
|
43 |
global con
|
| 43 |
if con is None:
|
44 |
if con is None:
|
| 44 |
print "Establishing connection %s host and port %d" %(host,port)
|
45 |
print "Establishing connection %s host and port %d" %(host,port)
|
| Line 91... |
Line 92... |
| 91 |
|
92 |
|
| 92 |
def returnLatestPrice(data, source_id, ignoreLastUpdated = True):
|
93 |
def returnLatestPrice(data, source_id, ignoreLastUpdated = True):
|
| 93 |
now = datetime.now()
|
94 |
now = datetime.now()
|
| 94 |
if source_id == 1:
|
95 |
if source_id == 1:
|
| 95 |
try:
|
96 |
try:
|
| 96 |
if data['identifier'] is None or len(data['identifier'].strip())==0:
|
97 |
if data['identifier'] is None or len(data['identifier'].strip()) !=10:
|
| 97 |
return {}
|
98 |
return {}
|
| 98 |
|
99 |
|
| 99 |
if data['dealFlag'] ==1 and data['dealType'] ==1:
|
100 |
if data['dealFlag'] ==1 and data['dealType'] ==1:
|
| 100 |
data['marketPlaceUrl'] = data['dealUrl'].strip()
|
101 |
data['marketPlaceUrl'] = data['dealUrl'].strip()
|
| 101 |
|
102 |
|
| Line 109... |
Line 110... |
| 109 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
|
110 |
return {'_id':data['_id'],'available_price':data['available_price'],'in_stock':data['in_stock'],'source_id':1,'source_product_name':data['source_product_name'],'marketPlaceUrl':data['marketPlaceUrl'],'thumbnail':data['thumbnail'],'coupon':data['coupon'], 'codAvailable':data['codAvailable'], 'tagline': data['tagline'], 'offer': data['offer']}
|
| 110 |
except:
|
111 |
except:
|
| 111 |
pass
|
112 |
pass
|
| 112 |
|
113 |
|
| 113 |
|
114 |
|
| 114 |
url = "http://www.amazon.in/gp/aw/ol/%s?o=New&op=1"%(data['identifier'])
|
- |
|
| 115 |
lowestPrice = 0.0
|
115 |
lowestPrice = 0.0
|
| 116 |
try:
|
- |
|
| 117 |
if data['dealFlag'] ==1 and data['dealType'] ==1:
|
- |
|
| 118 |
print "Inside deal"
|
- |
|
| 119 |
deal_url = data['marketPlaceUrl']
|
- |
|
| 120 |
print deal_url
|
- |
|
| 121 |
dealScraperAmazon = AmazonDealScraper.AmazonScraper(True)
|
116 |
asinPricingMap = amScraper.get_competitive_pricing_for_asin(marketplaceId, [data['identifier'].strip().upper()])
|
| 122 |
lowestPrice = dealScraperAmazon.read(deal_url)
|
- |
|
| 123 |
print lowestPrice
|
- |
|
| 124 |
if lowestPrice == 0:
|
- |
|
| 125 |
raise
|
- |
|
| 126 |
else:
|
- |
|
| 127 |
scraperAmazon = AmazonScraper(True)
|
- |
|
| 128 |
lowestPrice = scraperAmazon.read(url)
|
- |
|
| 129 |
except Exception as e:
|
- |
|
| 130 |
print e
|
- |
|
| 131 |
scraperAmazon = AmazonScraper(True)
|
- |
|
| 132 |
lowestPrice = scraperAmazon.read(url)
|
117 |
lowestPrice = asinPricingMap.get(data['identifier'].strip().upper())
|
| 133 |
print "LowestPrice ",lowestPrice
|
118 |
print "LowestPrice ",lowestPrice
|
| 134 |
inStock = 0
|
119 |
inStock = 0
|
| 135 |
if lowestPrice > 0:
|
120 |
if lowestPrice > 0:
|
| 136 |
inStock = 1
|
121 |
inStock = 1
|
| 137 |
if lowestPrice > 0:
|
122 |
if lowestPrice > 0:
|
| Line 697... |
Line 682... |
| 697 |
info = returnLatestPrice(item, item['source_id'],False)
|
682 |
info = returnLatestPrice(item, item['source_id'],False)
|
| 698 |
print info
|
683 |
print info
|
| 699 |
|
684 |
|
| 700 |
def main():
|
685 |
def main():
|
| 701 |
print datetime.now()
|
686 |
print datetime.now()
|
| 702 |
print "retuned %s"%(str(getLatestPrice(5899,4)))
|
687 |
print "returned %s"%(str(getLatestPriceById(23014)))
|
| 703 |
print datetime.now()
|
688 |
print datetime.now()
|
| 704 |
if __name__=='__main__':
|
689 |
if __name__=='__main__':
|
| 705 |
main()
|
690 |
main()
|
| 706 |
|
691 |
|