| Line 31... |
Line 31... |
| 31 |
import email
|
31 |
import email
|
| 32 |
from email.mime.multipart import MIMEMultipart
|
32 |
from email.mime.multipart import MIMEMultipart
|
| 33 |
import email.encoders
|
33 |
import email.encoders
|
| 34 |
import cookielib
|
34 |
import cookielib
|
| 35 |
from multiprocessing import Pool
|
35 |
from multiprocessing import Pool
|
| 36 |
from multiprocessing.dummy import Pool as ThreadPool
|
36 |
from multiprocessing.dummy import Pool as ThreadPool
|
| 37 |
from threading import Thread
|
37 |
import gc
|
| 38 |
|
38 |
|
| 39 |
config_client = ConfigClient()
|
39 |
config_client = ConfigClient()
|
| 40 |
host = config_client.get_property('staging_hostname')
|
40 |
host = config_client.get_property('staging_hostname')
|
| 41 |
syncPrice=config_client.get_property('sync_price_on_marketplace')
|
41 |
syncPrice=config_client.get_property('sync_price_on_marketplace')
|
| 42 |
|
42 |
|
| Line 2151... |
Line 2151... |
| 2151 |
else:
|
2151 |
else:
|
| 2152 |
count = 0
|
2152 |
count = 0
|
| 2153 |
pool = ThreadPool(50)
|
2153 |
pool = ThreadPool(50)
|
| 2154 |
startOffset = 0
|
2154 |
startOffset = 0
|
| 2155 |
endOffset = startOffset
|
2155 |
endOffset = startOffset
|
| 2156 |
threads = []
|
- |
|
| 2157 |
while(count<50 and endOffset<len(itemInfo)):
|
2156 |
while(count<1 and endOffset<len(itemInfo)):
|
| 2158 |
endOffset = startOffset + 1
|
2157 |
endOffset = startOffset + 50
|
| 2159 |
if (endOffset >= len(itemInfo)):
|
2158 |
if (endOffset >= len(itemInfo)):
|
| 2160 |
endOffset = len(itemInfo)
|
2159 |
endOffset = len(itemInfo)
|
| 2161 |
print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
|
2160 |
print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
|
| 2162 |
t = Thread(target=populateScrapingResults, args=(itemInfo[startOffset],))
|
- |
|
| 2163 |
t.start()
|
- |
|
| 2164 |
threads.append(t)
|
- |
|
| 2165 |
#pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
|
2161 |
pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
|
| 2166 |
#t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
|
2162 |
#t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
|
| 2167 |
#t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
|
2163 |
#t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
|
| 2168 |
#t = threading.Thread(target=partial(test, startOffset, endOffset))
|
2164 |
#t = threading.Thread(target=partial(test, startOffset, endOffset))
|
| 2169 |
#threads.append(t)
|
2165 |
#threads.append(t)
|
| 2170 |
startOffset = startOffset + 1
|
2166 |
startOffset = startOffset + 50
|
| 2171 |
count+=1
|
2167 |
count+=1
|
| 2172 |
#[t.start() for t in threads]
|
2168 |
#[t.start() for t in threads]
|
| 2173 |
#[t.join() for t in threads]
|
2169 |
#[t.join() for t in threads]
|
| 2174 |
#threads = []
|
2170 |
#threads = []
|
| 2175 |
print "terminating while"
|
2171 |
print "terminating while"
|
| 2176 |
# pool.close()
|
2172 |
pool.close()
|
| 2177 |
# pool.join()
|
2173 |
pool.join()
|
| 2178 |
for thread in threads:
|
- |
|
| 2179 |
thread.join()
|
- |
|
| 2180 |
print "joining threads"
|
2174 |
print "joining threads"
|
| 2181 |
print "returning offset******"
|
2175 |
print "returning offset******"
|
| 2182 |
return endOffset
|
2176 |
return endOffset
|
| 2183 |
|
2177 |
|
| 2184 |
def sendAutoPricingMail(successfulAutoDecrease,successfulAutoIncrease):
|
2178 |
def sendAutoPricingMail(successfulAutoDecrease,successfulAutoIncrease):
|
| Line 2294... |
Line 2288... |
| 2294 |
print "Error: unable to send email."
|
2288 |
print "Error: unable to send email."
|
| 2295 |
|
2289 |
|
| 2296 |
def processLostBuyBoxItems(previousProcessingTimestamp,currentTimestamp):
|
2290 |
def processLostBuyBoxItems(previousProcessingTimestamp,currentTimestamp):
|
| 2297 |
previous_buy_box = session.query(MarketPlaceHistory.item_id).filter(MarketPlaceHistory.timestamp==previousProcessingTimestamp).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).filter(or_(MarketPlaceHistory.competitiveCategory==CompetitionCategory.BUY_BOX,MarketPlaceHistory.competitiveCategory==CompetitionCategory.PREF_BUT_NOT_CHEAP)).all()
|
2291 |
previous_buy_box = session.query(MarketPlaceHistory.item_id).filter(MarketPlaceHistory.timestamp==previousProcessingTimestamp).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).filter(or_(MarketPlaceHistory.competitiveCategory==CompetitionCategory.BUY_BOX,MarketPlaceHistory.competitiveCategory==CompetitionCategory.PREF_BUT_NOT_CHEAP)).all()
|
| 2298 |
cant_compete = session.query(MarketPlaceHistory.item_id).filter(MarketPlaceHistory.timestamp==currentTimestamp).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).filter(MarketPlaceHistory.competitiveCategory==CompetitionCategory.CANT_COMPETE).all()
|
2292 |
cant_compete = session.query(MarketPlaceHistory.item_id).filter(MarketPlaceHistory.timestamp==currentTimestamp).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).filter(MarketPlaceHistory.competitiveCategory==CompetitionCategory.CANT_COMPETE).all()
|
| 2299 |
if previous_buy_box is None:
|
2293 |
if previous_buy_box is None or previous_buy_box==[]:
|
| 2300 |
print "No item in buy box for last run"
|
2294 |
print "No item in buy box for last run"
|
| 2301 |
return
|
2295 |
return
|
| 2302 |
lost_buy_box = list(set(list(zip(*previous_buy_box)[0]))&set(list(zip(*cant_compete)[0])))
|
2296 |
lost_buy_box = list(set(list(zip(*previous_buy_box)[0]))&set(list(zip(*cant_compete)[0])))
|
| 2303 |
if len(lost_buy_box)==0:
|
2297 |
if len(lost_buy_box)==0:
|
| 2304 |
return
|
2298 |
return
|
| Line 2668... |
Line 2662... |
| 2668 |
commitCompetitiveNoInventory(competitiveNoInventory,timestamp)
|
2662 |
commitCompetitiveNoInventory(competitiveNoInventory,timestamp)
|
| 2669 |
commitNegativeMargin(negativeMargin,timestamp)
|
2663 |
commitNegativeMargin(negativeMargin,timestamp)
|
| 2670 |
commitCheapButNotPref(cheapButNotPref,timestamp)
|
2664 |
commitCheapButNotPref(cheapButNotPref,timestamp)
|
| 2671 |
commitPrefButNotCheap(prefButNotCheap, timestamp)
|
2665 |
commitPrefButNotCheap(prefButNotCheap, timestamp)
|
| 2672 |
cantCompete[:], buyBoxItems[:], competitive[:], competitiveNoInventory[:], exceptionItems[:], negativeMargin[:], cheapButNotPref[:], prefButNotCheap[:] =[],[],[],[],[],[],[],[]
|
2666 |
cantCompete[:], buyBoxItems[:], competitive[:], competitiveNoInventory[:], exceptionItems[:], negativeMargin[:], cheapButNotPref[:], prefButNotCheap[:] =[],[],[],[],[],[],[],[]
|
| - |
|
2667 |
collected = gc.collect()
|
| - |
|
2668 |
print "Garbage collector: collected %d objects." % (collected)
|
| - |
|
2669 |
time.sleep(4)
|
| 2673 |
|
2670 |
|
| 2674 |
successfulAutoDecrease = fetchItemsForAutoDecrease(timestamp)
|
2671 |
successfulAutoDecrease = fetchItemsForAutoDecrease(timestamp)
|
| 2675 |
successfulAutoIncrease = fetchItemsForAutoIncrease(timestamp)
|
2672 |
successfulAutoIncrease = fetchItemsForAutoIncrease(timestamp)
|
| 2676 |
if options.runType=='FULL':
|
2673 |
if options.runType=='FULL':
|
| 2677 |
previousAutoFav, nowAutoFav = markAutoFavourite()
|
2674 |
previousAutoFav, nowAutoFav = markAutoFavourite()
|