| Line 42... |
Line 42... |
| 42 |
|
42 |
|
| 43 |
DataService.initialize(db_hostname=host)
|
43 |
DataService.initialize(db_hostname=host)
|
| 44 |
|
44 |
|
| 45 |
inventoryMap = {}
|
45 |
inventoryMap = {}
|
| 46 |
itemSaleMap = {}
|
46 |
itemSaleMap = {}
|
| 47 |
scraper = FlipkartScraper.FlipkartScraper()
|
- |
|
| 48 |
categoryMap = {}
|
47 |
categoryMap = {}
|
| 49 |
|
48 |
|
| 50 |
class __FlipkartDetails:
|
49 |
class __FlipkartDetails:
|
| 51 |
|
50 |
|
| 52 |
def __init__(self,rank ,ourSp , secondLowestSellerSp, prefSellerSp, lowestSellerSp, lowestSellerScore, prefSellerScore, secondLowestSellerScore, ourScore, shippingTimeLowerLimitLowestSeller,shippingTimeUpperLimitLowestSeller, \
|
51 |
def __init__(self,rank ,ourSp , secondLowestSellerSp, prefSellerSp, lowestSellerSp, lowestSellerScore, prefSellerScore, secondLowestSellerScore, ourScore, shippingTimeLowerLimitLowestSeller,shippingTimeUpperLimitLowestSeller, \
|
| Line 681... |
Line 680... |
| 681 |
flipkartItemInfo = __FlipkartItemInfo(flipkart_item.flipkartSerialNumber, flipkart_item.maxNlc,mp_item.courierCost, it.id, it.product_group, it.brand, it.model_name, it.model_number, it.color, it.weight, category.parent_category_id, it.risky, flipkart_item.warehouseId, None, runType, parent_category.display_name,sourcePercentage,None,flipkart_item.skuAtFlipkart,None,warehouse.stateId)
|
680 |
flipkartItemInfo = __FlipkartItemInfo(flipkart_item.flipkartSerialNumber, flipkart_item.maxNlc,mp_item.courierCost, it.id, it.product_group, it.brand, it.model_name, it.model_number, it.color, it.weight, category.parent_category_id, it.risky, flipkart_item.warehouseId, None, runType, parent_category.display_name,sourcePercentage,None,flipkart_item.skuAtFlipkart,None,warehouse.stateId)
|
| 682 |
itemInfo.append(flipkartItemInfo)
|
681 |
itemInfo.append(flipkartItemInfo)
|
| 683 |
session.close()
|
682 |
session.close()
|
| 684 |
return itemInfo
|
683 |
return itemInfo
|
| 685 |
|
684 |
|
| 686 |
def fetchDetails(flipkartSerialNumber,scraper):
|
685 |
def fetchDetails(flipkartSerialNumber):
|
| 687 |
url = "http://www.flipkart.com/ps/%s"%(flipkartSerialNumber)
|
686 |
url = "http://www.flipkart.com/ps/%s"%(flipkartSerialNumber)
|
| 688 |
#url = "http://www.flipkart.com/ps/MOBDTXVZXVY3GFG8"
|
687 |
#url = "http://www.flipkart.com/ps/MOBDTXVZXVY3GFG8"
|
| 689 |
#scraper.read(url)
|
688 |
#scraper.read(url)
|
| - |
|
689 |
scraper = FlipkartScraper.FlipkartScraper()
|
| 690 |
vendorsData = scraper.read(url)
|
690 |
vendorsData = scraper.read(url)
|
| 691 |
fin = datetime.now()
|
691 |
fin = datetime.now()
|
| 692 |
print "Finish with data for serial Number %s %s" %(flipkartSerialNumber,str(fin))
|
692 |
print "Finish with data for serial Number %s %s" %(flipkartSerialNumber,str(fin))
|
| 693 |
sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
|
693 |
sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
|
| 694 |
vendorsData[:]=[]
|
694 |
vendorsData[:]=[]
|
| Line 2101... |
Line 2101... |
| 2101 |
|
2101 |
|
| 2102 |
def populateScrapingResults(val):
|
2102 |
def populateScrapingResults(val):
|
| 2103 |
try:
|
2103 |
try:
|
| 2104 |
now = datetime.now()
|
2104 |
now = datetime.now()
|
| 2105 |
print "Fetching data for serial Number %s %s" %(val.fkSerialNumber,str(now))
|
2105 |
print "Fetching data for serial Number %s %s" %(val.fkSerialNumber,str(now))
|
| 2106 |
flipkartDetails = fetchDetails(val.fkSerialNumber,scraper)
|
2106 |
flipkartDetails = fetchDetails(val.fkSerialNumber)
|
| 2107 |
val.flipkartDetails = flipkartDetails
|
2107 |
val.flipkartDetails = flipkartDetails
|
| 2108 |
except Exception as e:
|
2108 |
except Exception as e:
|
| 2109 |
print "Unable to fetch details of %s" %(val.fkSerialNumber)
|
2109 |
print "Unable to fetch details of %s" %(val.fkSerialNumber)
|
| 2110 |
print e
|
2110 |
print e
|
| 2111 |
val.flipkartDetails = None
|
2111 |
val.flipkartDetails = None
|
| Line 2121... |
Line 2121... |
| 2121 |
# finally:
|
2121 |
# finally:
|
| 2122 |
# r={}
|
2122 |
# r={}
|
| 2123 |
#
|
2123 |
#
|
| 2124 |
val.ourFlipkartInventory = 0
|
2124 |
val.ourFlipkartInventory = 0
|
| 2125 |
|
2125 |
|
| 2126 |
def threadsToSpawn(runType,itemInfo,scraper,itemPopulated):
|
2126 |
def threadsToSpawn(runType,itemInfo,itemPopulated):
|
| 2127 |
if runType == RunType.FAVOURITE:
|
2127 |
if runType == RunType.FAVOURITE:
|
| 2128 |
count = 0
|
2128 |
count = 0
|
| 2129 |
pool = ThreadPool(3)
|
2129 |
pool = ThreadPool(3)
|
| 2130 |
startOffset = 0
|
2130 |
startOffset = 0
|
| 2131 |
endOffset = startOffset
|
2131 |
endOffset = startOffset
|
| Line 2649... |
Line 2649... |
| 2649 |
timestamp = datetime.now()
|
2649 |
timestamp = datetime.now()
|
| 2650 |
previousProcessingTimestamp = session.query(func.max(MarketPlaceHistory.timestamp)).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).one()
|
2650 |
previousProcessingTimestamp = session.query(func.max(MarketPlaceHistory.timestamp)).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).one()
|
| 2651 |
itemInfo= populateStuff(options.runType,timestamp)
|
2651 |
itemInfo= populateStuff(options.runType,timestamp)
|
| 2652 |
itemsPopulated = 0
|
2652 |
itemsPopulated = 0
|
| 2653 |
while (len(itemInfo)>0):
|
2653 |
while (len(itemInfo)>0):
|
| 2654 |
itemsPopulated = threadsToSpawn(options.runType,itemInfo,scraper,itemsPopulated)
|
2654 |
itemsPopulated = threadsToSpawn(options.runType,itemInfo,itemsPopulated)
|
| 2655 |
cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = decideCategory(itemInfo[0:itemsPopulated])
|
2655 |
cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = decideCategory(itemInfo[0:itemsPopulated])
|
| 2656 |
itemInfo[0:itemsPopulated] = []
|
2656 |
itemInfo[0:itemsPopulated] = []
|
| 2657 |
commitExceptionList(exceptionItems,timestamp)
|
2657 |
commitExceptionList(exceptionItems,timestamp)
|
| 2658 |
commitCantCompete(cantCompete,timestamp)
|
2658 |
commitCantCompete(cantCompete,timestamp)
|
| 2659 |
commitBuyBox(buyBoxItems,timestamp)
|
2659 |
commitBuyBox(buyBoxItems,timestamp)
|