WebSVN – SmartDukaan – Diff – //trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py

         self.secondLowestSellerBuyTrend = secondLowestSellerBuyTrend
         self.ourCode = ourCode
 class __FlipkartItemInfo:
-    def __init__(self, fkSerialNumber, nlc, courierCost, item_id, product_group, brand, model_name, model_number, color, weight, parent_category, risky, warehouseId, vatRate, runType, parent_category_name, sourcePercentage, ourFlipkartInventory, skuAtFlipkart, flipkartDetails):
+    def __init__(self, fkSerialNumber, nlc, courierCost, item_id, product_group, brand, model_name, model_number, color, weight, parent_category, risky, warehouseId, vatRate, runType, parent_category_name, sourcePercentage, ourFlipkartInventory, skuAtFlipkart, flipkartDetails, stateId):
         self.fkSerialNumber = fkSerialNumber
         self.nlc = nlc
         self.courierCost = courierCost
         self.item_id = item_id
         self.runType = runType
         self.parent_category_name = parent_category_name
         self.sourcePercentage = sourcePercentage
         self.ourFlipkartInventory = ourFlipkartInventory
         self.skuAtFlipkart = skuAtFlipkart
         self.flipkartDetails = flipkartDetails
+        self.stateId = stateId
 class __FlipkartPricing:
     def __init__(self, ourSp, ourTp, lowestTp, lowestPossibleTp, secondLowestSellerTp, lowestPossibleSp, prefSellerTp):
         self.ourTp = ourTp
         prefButNotCheapItems.append(mpHistory)
     session.commit()
     return prefButNotCheapItems
 def populateStuff(runType,time):
+    global itemSaleMap
     itemInfo = []
     if runType=='FAVOURITE':
         items = session.query(FlipkartItem,MarketplaceItems).join((MarketplaceItems,FlipkartItem.item_id==MarketplaceItems.itemId)).filter(MarketplaceItems.source==OrderSource.FLIPKART).\
         filter(or_(MarketplaceItems.autoFavourite==True, MarketplaceItems.manualFavourite==True)).all()
     else:
         #items = session.query(FlipkartItem,MarketplaceItems).join((MarketplaceItems,FlipkartItem.item_id==MarketplaceItems.itemId)).filter(MarketplaceItems.source==OrderSource.FLIPKART).all()
         items = session.query(FlipkartItem,MarketplaceItems).join((MarketplaceItems,FlipkartItem.item_id==MarketplaceItems.itemId)).filter(MarketplaceItems.source==OrderSource.FLIPKART).all()
+    inventory_client = InventoryClient().get_client()
     for item in items:
         flipkart_item = item[0]
         mp_item = item[1]
         it = Item.query.filter_by(id=flipkart_item.item_id).one()
         category = Category.query.filter_by(id=it.category).one()
             if scp is not None:
                 sourcePercentage = scp
             else:
                 spm = SourcePercentageMaster.get_by(source=OrderSource.FLIPKART)
                 sourcePercentage = spm
+        warehouse = inventory_client.getWarehouse(flipkart_item.warehouseId)
+        itemSaleList = []
+        oosForAllSources = inventory_client.getOosStatusesForXDaysForItem(flipkart_item.item_id, 0, 3)
+        oosForFlipkart = inventory_client.getOosStatusesForXDaysForItem(flipkart_item.item_id, OrderSource.FLIPKART, 5)
+        oosForFlipkartLastDay = inventory_client.getOosStatusesForXDaysForItem(flipkart_item.item_id, OrderSource.FLIPKART, 1)
+        itemSaleList.append(oosForAllSources)
+        itemSaleList.append(oosForFlipkart)
+        itemSaleList.append(calculateAverageSale(oosForAllSources))
+        itemSaleList.append(calculateAverageSale(oosForFlipkart))
+        itemSaleList.append(calculateAverageSale(oosForFlipkartLastDay))
+        itemSaleList.append(calculateTotalSale(oosForFlipkart))
+        itemSaleMap[flipkart_item.item_id]=itemSaleList
 #        try:
 #            request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(flipkart_item.skuAtFlipkart))
 #            r = requests.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
 #            print "Inventory info",r.json()
 #            stock_count = int((r.json()['attributeValues'])['stock_count'])
 #        except:
 #            stock_count = 0
-        flipkartItemInfo = __FlipkartItemInfo(flipkart_item.flipkartSerialNumber, flipkart_item.maxNlc,mp_item.courierCost, it.id, it.product_group, it.brand, it.model_name, it.model_number, it.color, it.weight, category.parent_category_id, it.risky, flipkart_item.warehouseId, None, runType, parent_category.display_name,sourcePercentage,None,flipkart_item.skuAtFlipkart,None)
+        flipkartItemInfo = __FlipkartItemInfo(flipkart_item.flipkartSerialNumber, flipkart_item.maxNlc,mp_item.courierCost, it.id, it.product_group, it.brand, it.model_name, it.model_number, it.color, it.weight, category.parent_category_id, it.risky, flipkart_item.warehouseId, None, runType, parent_category.display_name,sourcePercentage,None,flipkart_item.skuAtFlipkart,None,warehouse.stateId)
         itemInfo.append(flipkartItemInfo)
     return itemInfo
 def fetchDetails(flipkartSerialNumber,scraper):
     url = "http://www.flipkart.com/ps/%s"%(flipkartSerialNumber)
     #url = "http://www.flipkart.com/ps/MOBDTXVZXVY3GFG8"
     scraper.read(url)
     vendorsData = scraper.createData()
     print "Vendor data for flipkartSerialNumber",flipkartSerialNumber
     print "Json Data",vendorsData
+    sortedVendorsData = []
     sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
     rank ,ourSp, iterator, secondLowestSellerSp, prefSellerSp, lowestSellerSp, lowestSellerScore, prefSellerScore, secondLowestSellerScore, ourScore, shippingTimeLowerLimitLowestSeller,shippingTimeUpperLimitLowestSeller, \
     shippingTimeLowerLimitPrefSeller, shippingTimeUpperLimitPrefSeller, shippingTimeLowerLimitOur, shippingTimeUpperLimitOur, shippingTimeLowerLimitSecondLowestSeller, shippingTimeUpperLimitSecondLowestSeller, totalAvailableSeller= (0,)*19
     lowestSellerName, lowestSellerCode, secondLowestSellerName, secondLowestSellerCode, prefSellerName, prefSellerCode, lowestSellerBuyTrend, \
     ourBuyTrend, prefSellerBuyTrend, secondLowestSellerBuyTrend, ourCode = ('',)*11
     global itemSaleMap
     cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = [],[],[],[],[],[],[],[]
     catalog_client = CatalogClient().get_client()
-    inventory_client = InventoryClient().get_client()
     for val in itemInfo:
         spm = val.sourcePercentage
         flipkartDetails = val.flipkartDetails
-        #print "Fetching details of ",val.fkSerialNumber
-        #flipkartDetails = fetchDetails(val.fkSerialNumber,scraper)
-        if (flipkartDetails.totalAvailableSeller==0):
+        if (flipkartDetails is None or flipkartDetails.totalAvailableSeller==0):
             exceptionItems.append(val)
             continue
         mpItem = MarketplaceItems.get_by(itemId=val.item_id,source=OrderSource.FLIPKART)
-        warehouse = inventory_client.getWarehouse(val.warehouseId)
-        itemSaleList = []
-        oosForAllSources = inventory_client.getOosStatusesForXDaysForItem(val.item_id, 0, 3)
-        oosForFlipkart = inventory_client.getOosStatusesForXDaysForItem(val.item_id, OrderSource.FLIPKART, 5)
-        oosForFlipkartLastDay = inventory_client.getOosStatusesForXDaysForItem(val.item_id, OrderSource.FLIPKART, 1)
-        itemSaleList.append(oosForAllSources)
-        itemSaleList.append(oosForFlipkart)
-        itemSaleList.append(calculateAverageSale(oosForAllSources))
-        itemSaleList.append(calculateAverageSale(oosForFlipkart))
-        itemSaleList.append(calculateAverageSale(oosForFlipkartLastDay))
-        itemSaleList.append(calculateTotalSale(oosForFlipkart))
-        itemSaleMap[val.item_id]=itemSaleList
         if flipkartDetails.rank==0:
             flipkartDetails.ourSp = mpItem.currentSp
             ourSp = mpItem.currentSp
         else:
             ourSp = flipkartDetails.ourSp
-        vatRate = catalog_client.getVatPercentageForItem(val.item_id, warehouse.stateId, flipkartDetails.ourSp)
+        vatRate = catalog_client.getVatPercentageForItem(val.item_id, val.stateId, flipkartDetails.ourSp)
         val.vatRate = vatRate
         if (flipkartDetails.ourBuyTrend == 'PrefCheap') or (flipkartDetails.rank==1 and flipkartDetails.totalAvailableSeller==1):
             temp=[]
             temp.append(flipkartDetails)
             temp.append(val)
         threads = []
         return endOffset
     else:
         count = 0
         pool = ThreadPool(5)
-        startOffset = itemPopulated
+        startOffset = 0
         endOffset = startOffset
         while(count<5 and endOffset<len(itemInfo)):
             print "inside while",testcount
-            endOffset = startOffset + 50
+            endOffset = startOffset + 20
             if (endOffset >= len(itemInfo)):
                 endOffset = len(itemInfo)
+            print "pool offset start end count"+str(startOffset)+" "+str(endOffset)+" "+str(count)
-            print itemInfo[startOffset:endOffset]
+            time.sleep(10)
             pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
             #t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
             #t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
             #t = threading.Thread(target=partial(test, startOffset, endOffset))
             #threads.append(t)
-            startOffset = startOffset + 50
+            startOffset = startOffset + 20
             count+=1
         #[t.start() for t in threads]
         #[t.join() for t in threads]
         #threads = []
+        print "terminating while"
         pool.close()
         pool.join()
+        print "joining threads"
+        time.sleep(5)
-        "returning offset******"
+        print "returning offset******"
         time.sleep(5)
         return endOffset
 def main():
     (options, args) = parser.parse_args()
     if options.runType not in ('FULL','FAVOURITE'):
         print "Run type argument illegal."
         sys.exit(1)
     timestamp = datetime.now()
+    previousProcessingTimestamp = session.query(func.max(MarketPlaceHistory.timestamp)).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).one()
     itemInfo= populateStuff(options.runType,timestamp)
     itemsPopulated = 0
     while (itemsPopulated<len(itemInfo)):
         itemsPopulated = threadsToSpawn(options.runType,itemInfo,scraper,itemsPopulated)
-    previousProcessingTimestamp = session.query(func.max(MarketPlaceHistory.timestamp)).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).one()
-    cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = decideCategory(itemInfo)
+        cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = decideCategory(itemInfo[0:itemsPopulated])
+        itemInfo[0:itemsPopulated] = []
-    commitExceptionList(exceptionItems,timestamp)
+        commitExceptionList(exceptionItems,timestamp)
-    commitCantCompete(cantCompete,timestamp)
+        commitCantCompete(cantCompete,timestamp)
-    commitBuyBox(buyBoxItems,timestamp)
+        commitBuyBox(buyBoxItems,timestamp)
-    commitCompetitive(competitive,timestamp)
+        commitCompetitive(competitive,timestamp)
-    commitCompetitiveNoInventory(competitiveNoInventory,timestamp)
+        commitCompetitiveNoInventory(competitiveNoInventory,timestamp)
-    commitNegativeMargin(negativeMargin,timestamp)
+        commitNegativeMargin(negativeMargin,timestamp)
-    commitCheapButNotPref(cheapButNotPref,timestamp)
+        commitCheapButNotPref(cheapButNotPref,timestamp)
-    commitPrefButNotCheap(prefButNotCheap, timestamp)
+        commitPrefButNotCheap(prefButNotCheap, timestamp)
+        cantCompete[:], buyBoxItems[:], competitive[:], competitiveNoInventory[:], exceptionItems[:], negativeMargin[:], cheapButNotPref[:], prefButNotCheap[:] =[],[],[],[],[],[],[],[]
-    #getLastUpdatedOn(mailSender.start, mailSender.stop)
+        #getLastUpdatedOn(mailSender.start, mailSender.stop)
     #successfulAutoDecrease = fetchItemsForAutoDecrease(timestamp)
     #successfulAutoIncrease = fetchItemsForAutoIncrease(timestamp)
-    if options.runType=='FULL':
+#    if options.runType=='FULL':
-        previousAutoFav, nowAutoFav = markAutoFavourite()
+#        previousAutoFav, nowAutoFav = markAutoFavourite()
-    if options.runType=='FULL':
+#    if options.runType=='FULL':
-        write_report(cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap, previousAutoFav, nowAutoFav,timestamp, options.runType)
+#        write_report(cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap, previousAutoFav, nowAutoFav,timestamp, options.runType)
-    else:
+#    else:
-        write_report(cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap, None, None, timestamp, options.runType)
+#        write_report(cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap, None, None, timestamp, options.runType)
 if __name__ == '__main__':
     main()

Subversion Repositories SmartDukaan

(root)//trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py – Rev 11571 → 11581