WebSVN – SmartDukaan – Diff – //trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py

 syncPrice=config_client.get_property('sync_price_on_marketplace')
 DataService.initialize(db_hostname=host)
-threads = []
 inventoryMap = {}
 itemSaleMap = {}
-cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = [],[],[],[],[],[],[],[]
 scraper = FlipkartScraper.FlipkartScraper()
+testcount = 0
 class __FlipkartDetails:
     def __init__(self,rank ,ourSp , secondLowestSellerSp, prefSellerSp, lowestSellerSp, lowestSellerScore, prefSellerScore, secondLowestSellerScore, ourScore, shippingTimeLowerLimitLowestSeller,shippingTimeUpperLimitLowestSeller, \
     shippingTimeLowerLimitPrefSeller, shippingTimeUpperLimitPrefSeller, shippingTimeLowerLimitOur, shippingTimeUpperLimitOur, shippingTimeLowerLimitSecondLowestSeller, shippingTimeUpperLimitSecondLowestSeller, totalAvailableSeller, lowestSellerName, lowestSellerCode, secondLowestSellerName, secondLowestSellerCode, prefSellerName, prefSellerCode, lowestSellerBuyTrend, \
         self.secondLowestSellerBuyTrend = secondLowestSellerBuyTrend
         self.ourCode = ourCode
 class __FlipkartItemInfo:
-    def __init__(self, fkSerialNumber, nlc, courierCost, item_id, product_group, brand, model_name, model_number, color, weight, parent_category, risky, warehouseId, vatRate, runType, parent_category_name, sourcePercentage, ourFlipkartInventory, skuAtFlipkart):
+    def __init__(self, fkSerialNumber, nlc, courierCost, item_id, product_group, brand, model_name, model_number, color, weight, parent_category, risky, warehouseId, vatRate, runType, parent_category_name, sourcePercentage, ourFlipkartInventory, skuAtFlipkart, flipkartDetails):
         self.fkSerialNumber = fkSerialNumber
         self.nlc = nlc
         self.courierCost = courierCost
         self.item_id = item_id
         self.vatRate = vatRate
         self.runType = runType
         self.parent_category_name = parent_category_name
         self.sourcePercentage = sourcePercentage
         self.ourFlipkartInventory = ourFlipkartInventory
         self.skuAtFlipkart = skuAtFlipkart
+        self.flipkartDetails = flipkartDetails
 class __FlipkartPricing:
     def __init__(self, ourSp, ourTp, lowestTp, lowestPossibleTp, secondLowestSellerTp, lowestPossibleSp, prefSellerTp):
         self.ourTp = ourTp
 #            r = requests.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
 #            print "Inventory info",r.json()
 #            stock_count = int((r.json()['attributeValues'])['stock_count'])
 #        except:
 #            stock_count = 0
-        flipkartItemInfo = __FlipkartItemInfo(flipkart_item.flipkartSerialNumber, flipkart_item.maxNlc,mp_item.courierCost, it.id, it.product_group, it.brand, it.model_name, it.model_number, it.color, it.weight, category.parent_category_id, it.risky, flipkart_item.warehouseId, None, runType, parent_category.display_name,sourcePercentage,None,flipkart_item.skuAtFlipkart)
+        flipkartItemInfo = __FlipkartItemInfo(flipkart_item.flipkartSerialNumber, flipkart_item.maxNlc,mp_item.courierCost, it.id, it.product_group, it.brand, it.model_name, it.model_number, it.color, it.weight, category.parent_category_id, it.risky, flipkart_item.warehouseId, None, runType, parent_category.display_name,sourcePercentage,None,flipkart_item.skuAtFlipkart,None)
         itemInfo.append(flipkartItemInfo)
     return itemInfo
 def fetchDetails(flipkartSerialNumber,scraper):
     url = "http://www.flipkart.com/ps/%s"%(flipkartSerialNumber)
     elif (float(lowestSellingPrice - ourNlc))/lowestSellingPrice >=0 and (float(lowestSellingPrice - ourNlc))/lowestSellingPrice <=.02:
         return 'MEDIUM'
     else:
         return 'LOW'
-def decideCategory(val):
+def decideCategory(itemInfo):
-    print type(val)
+    print "length****",len(itemInfo)
-    print val
-    print "dusdlsdkskd"
     global itemSaleMap
-    global cantCompete
-    global buyBoxItems
-    global competitive
-    global competitiveNoInventory
+    cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = [],[],[],[],[],[],[],[]
-    global exceptionItems
-    global negativeMargin
-    global cheapButNotPref
-    global prefButNotCheap
     catalog_client = CatalogClient().get_client()
     inventory_client = InventoryClient().get_client()
-    spm = val.sourcePercentage
+    for val in itemInfo:
-    #print "Fetching details of ",val.fkSerialNumber
+        spm = val.sourcePercentage
-    #flipkartDetails = fetchDetails(val.fkSerialNumber,scraper)
+        flipkartDetails = val.flipkartDetails
-    try:
+        #print "Fetching details of ",val.fkSerialNumber
-        flipkartDetails = fetchDetails(val.fkSerialNumber,scraper)
+        #flipkartDetails = fetchDetails(val.fkSerialNumber,scraper)
-    except Exception as e:
-        print "Unable to fetch details of",val.fkSerialNumber
+        if (flipkartDetails.totalAvailableSeller==0):
-        print e
+            exceptionItems.append(val)
-        exceptionItems.append(val)
+            continue
-        return
+        mpItem = MarketplaceItems.get_by(itemId=val.item_id,source=OrderSource.FLIPKART)
-    try:
+        warehouse = inventory_client.getWarehouse(val.warehouseId)
-        request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(val.skuAtFlipkart))
-        r = requests.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
+        itemSaleList = []
-        print "Inventory info",r.json()
+        oosForAllSources = inventory_client.getOosStatusesForXDaysForItem(val.item_id, 0, 3)
-        stock_count = int((r.json()['attributeValues'])['stock_count'])
+        oosForFlipkart = inventory_client.getOosStatusesForXDaysForItem(val.item_id, OrderSource.FLIPKART, 5)
-    except:
+        oosForFlipkartLastDay = inventory_client.getOosStatusesForXDaysForItem(val.item_id, OrderSource.FLIPKART, 1)
-        stock_count = 0
+        itemSaleList.append(oosForAllSources)
+        itemSaleList.append(oosForFlipkart)
-    val.ourFlipkartInventory = stock_count
+        itemSaleList.append(calculateAverageSale(oosForAllSources))
+        itemSaleList.append(calculateAverageSale(oosForFlipkart))
-    if (flipkartDetails.totalAvailableSeller==0):
+        itemSaleList.append(calculateAverageSale(oosForFlipkartLastDay))
-        exceptionItems.append(val)
+        itemSaleList.append(calculateTotalSale(oosForFlipkart))
-        return
+        itemSaleMap[val.item_id]=itemSaleList
-    mpItem = MarketplaceItems.get_by(itemId=val.item_id,source=OrderSource.FLIPKART)
+        if flipkartDetails.rank==0:
-    warehouse = inventory_client.getWarehouse(val.warehouseId)
+            flipkartDetails.ourSp = mpItem.currentSp
+            ourSp = mpItem.currentSp
-    itemSaleList = []
+        else:
-    oosForAllSources = inventory_client.getOosStatusesForXDaysForItem(val.item_id, 0, 3)
+            ourSp = flipkartDetails.ourSp
-    oosForFlipkart = inventory_client.getOosStatusesForXDaysForItem(val.item_id, OrderSource.FLIPKART, 5)
+        vatRate = catalog_client.getVatPercentageForItem(val.item_id, warehouse.stateId, flipkartDetails.ourSp)
-    oosForFlipkartLastDay = inventory_client.getOosStatusesForXDaysForItem(val.item_id, OrderSource.FLIPKART, 1)
+        val.vatRate = vatRate
-    itemSaleList.append(oosForAllSources)
+        if (flipkartDetails.ourBuyTrend == 'PrefCheap') or (flipkartDetails.rank==1 and flipkartDetails.totalAvailableSeller==1):
-    itemSaleList.append(oosForFlipkart)
+            temp=[]
-    itemSaleList.append(calculateAverageSale(oosForAllSources))
+            temp.append(flipkartDetails)
-    itemSaleList.append(calculateAverageSale(oosForFlipkart))
+            temp.append(val)
-    itemSaleList.append(calculateAverageSale(oosForFlipkartLastDay))
+            secondLowestTp=0 if flipkartDetails.totalAvailableSeller==1 else getOtherTp(flipkartDetails,val,spm,False)
-    itemSaleList.append(calculateTotalSale(oosForFlipkart))
+            prefSellerTp=0 if flipkartDetails.totalAvailableSeller < 2 else getOtherTp(flipkartDetails,val,spm,True)
-    itemSaleMap[val.item_id]=itemSaleList
+            flipkartPricing = __FlipkartPricing(flipkartDetails.ourSp,getOurTp(flipkartDetails,val,spm,mpItem),None,getLowestPossibleTp(flipkartDetails,val,spm,mpItem),secondLowestTp,getLowestPossibleSp(flipkartDetails,val,spm,mpItem),prefSellerTp)
+            temp.append(flipkartPricing)
-    if flipkartDetails.rank==0:
+            temp.append(mpItem)
-        flipkartDetails.ourSp = mpItem.currentSp
+            buyBoxItems.append(temp)
-        ourSp = mpItem.currentSp
+            continue
-    else:
-        ourSp = flipkartDetails.ourSp
+        if (flipkartDetails.ourBuyTrend == 'PrefNCheap'):
-    vatRate = catalog_client.getVatPercentageForItem(val.item_id, warehouse.stateId, flipkartDetails.ourSp)
+            temp=[]
-    val.vatRate = vatRate
+            temp.append(flipkartDetails)
-    if (flipkartDetails.ourBuyTrend == 'PrefCheap') or (flipkartDetails.rank==1 and flipkartDetails.totalAvailableSeller==1):
+            temp.append(val)
-        temp=[]
+            secondLowestTp=0 if flipkartDetails.totalAvailableSeller==1 else getOtherTp(flipkartDetails,val,spm,False)
-        temp.append(flipkartDetails)
+            prefSellerTp=0 if flipkartDetails.totalAvailableSeller < 2 else getOtherTp(flipkartDetails,val,spm,True)
-        temp.append(val)
+            flipkartPricing = __FlipkartPricing(flipkartDetails.ourSp,getOurTp(flipkartDetails,val,spm,mpItem),None,getLowestPossibleTp(flipkartDetails,val,spm,mpItem),secondLowestTp,getLowestPossibleSp(flipkartDetails,val,spm,mpItem),prefSellerTp)
-        secondLowestTp=0 if flipkartDetails.totalAvailableSeller==1 else getOtherTp(flipkartDetails,val,spm,False)
+            temp.append(flipkartPricing)
-        prefSellerTp=0 if flipkartDetails.totalAvailableSeller < 2 else getOtherTp(flipkartDetails,val,spm,True)
+            temp.append(mpItem)
-        flipkartPricing = __FlipkartPricing(flipkartDetails.ourSp,getOurTp(flipkartDetails,val,spm,mpItem),None,getLowestPossibleTp(flipkartDetails,val,spm,mpItem),secondLowestTp,getLowestPossibleSp(flipkartDetails,val,spm,mpItem),prefSellerTp)
+            prefButNotCheap.append(temp)
-        temp.append(flipkartPricing)
+            continue
-        temp.append(mpItem)
-        buyBoxItems.append(temp)
+        if (flipkartDetails.ourBuyTrend == 'NPrefCheap') and (flipkartDetails.rank==1):
-        return
+            temp=[]
+            temp.append(flipkartDetails)
-    if (flipkartDetails.ourBuyTrend == 'PrefNCheap'):
+            temp.append(val)
-        temp=[]
+            prefSellerTp=0 if flipkartDetails.totalAvailableSeller < 2 else getOtherTp(flipkartDetails,val,spm,True)
-        temp.append(flipkartDetails)
+            flipkartPricing = __FlipkartPricing(flipkartDetails.ourSp,getOurTp(flipkartDetails,val,spm,mpItem),None,getLowestPossibleTp(flipkartDetails,val,spm,mpItem),None,getLowestPossibleSp(flipkartDetails,val,spm,mpItem),prefSellerTp)
-        temp.append(val)
+            temp.append(flipkartPricing)
-        secondLowestTp=0 if flipkartDetails.totalAvailableSeller==1 else getOtherTp(flipkartDetails,val,spm,False)
+            temp.append(mpItem)
-        prefSellerTp=0 if flipkartDetails.totalAvailableSeller < 2 else getOtherTp(flipkartDetails,val,spm,True)
+            cheapButNotPref.append(temp)
-        flipkartPricing = __FlipkartPricing(flipkartDetails.ourSp,getOurTp(flipkartDetails,val,spm,mpItem),None,getLowestPossibleTp(flipkartDetails,val,spm,mpItem),secondLowestTp,getLowestPossibleSp(flipkartDetails,val,spm,mpItem),prefSellerTp)
+            continue
-        temp.append(flipkartPricing)
-        temp.append(mpItem)
-        prefButNotCheap.append(temp)
+        lowestTp = getOtherTp(flipkartDetails,val,spm,False)
-        return
+        ourTp = getOurTp(flipkartDetails,val,spm,mpItem)
+        lowestPossibleTp = getLowestPossibleTp(flipkartDetails,val,spm,mpItem)
-    if (flipkartDetails.ourBuyTrend == 'NPrefCheap') and (flipkartDetails.rank==1):
+        lowestPossibleSp = getLowestPossibleSp(flipkartDetails,val,spm,mpItem)
-        temp=[]
+        prefSellerTp = getOtherTp(flipkartDetails,val,spm,True)
-        temp.append(flipkartDetails)
-        temp.append(val)
+        if (ourTp<lowestPossibleTp):
-        prefSellerTp=0 if flipkartDetails.totalAvailableSeller < 2 else getOtherTp(flipkartDetails,val,spm,True)
+            temp=[]
-        flipkartPricing = __FlipkartPricing(flipkartDetails.ourSp,getOurTp(flipkartDetails,val,spm,mpItem),None,getLowestPossibleTp(flipkartDetails,val,spm,mpItem),None,getLowestPossibleSp(flipkartDetails,val,spm,mpItem),prefSellerTp)
+            temp.append(flipkartDetails)
-        temp.append(flipkartPricing)
+            temp.append(val)
-        temp.append(mpItem)
+            flipkartPricing = __FlipkartPricing(ourSp,ourTp,lowestTp,lowestPossibleTp,None,None,None)
-        cheapButNotPref.append(temp)
+            temp.append(flipkartPricing)
-        return
+            negativeMargin.append(temp)
+            continue
-    lowestTp = getOtherTp(flipkartDetails,val,spm,False)
+        if (flipkartDetails.lowestSellerSp > lowestPossibleSp) and val.ourFlipkartInventory!=0:
-    ourTp = getOurTp(flipkartDetails,val,spm,mpItem)
+            type(val.ourFlipkartInventory)
-    lowestPossibleTp = getLowestPossibleTp(flipkartDetails,val,spm,mpItem)
+            temp=[]
-    lowestPossibleSp = getLowestPossibleSp(flipkartDetails,val,spm,mpItem)
+            temp.append(flipkartDetails)
-    prefSellerTp = getOtherTp(flipkartDetails,val,spm,True)
+            temp.append(val)
+            flipkartPricing = __FlipkartPricing(ourSp,ourTp,lowestTp,lowestPossibleTp,None,lowestPossibleSp,prefSellerTp)
-    if (ourTp<lowestPossibleTp):
+            temp.append(flipkartPricing)
-        temp=[]
+            temp.append(mpItem)
-        temp.append(flipkartDetails)
+            competitive.append(temp)
-        temp.append(val)
+            continue
-        flipkartPricing = __FlipkartPricing(ourSp,ourTp,lowestTp,lowestPossibleTp,None,None,None)
-        temp.append(flipkartPricing)
+        if (flipkartDetails.lowestSellerSp) > lowestPossibleSp and val.ourFlipkartInventory==0:
-        negativeMargin.append(temp)
+            temp=[]
-        return
+            temp.append(flipkartDetails)
+            temp.append(val)
-    if (flipkartDetails.lowestSellerSp > lowestPossibleSp) and val.ourFlipkartInventory!=0:
+            flipkartPricing = __FlipkartPricing(ourSp,ourTp,lowestTp,lowestPossibleTp,None,lowestPossibleSp,prefSellerTp)
-        type(val.ourFlipkartInventory)
+            temp.append(flipkartPricing)
-        temp=[]
+            temp.append(mpItem)
-        temp.append(flipkartDetails)
+            competitiveNoInventory.append(temp)
-        temp.append(val)
+            continue
-        flipkartPricing = __FlipkartPricing(ourSp,ourTp,lowestTp,lowestPossibleTp,None,lowestPossibleSp,prefSellerTp)
-        temp.append(flipkartPricing)
-        temp.append(mpItem)
-        competitive.append(temp)
-        return
-    if (flipkartDetails.lowestSellerSp) > lowestPossibleSp and val.ourFlipkartInventory==0:
         temp=[]
         temp.append(flipkartDetails)
         temp.append(val)
         flipkartPricing = __FlipkartPricing(ourSp,ourTp,lowestTp,lowestPossibleTp,None,lowestPossibleSp,prefSellerTp)
         temp.append(flipkartPricing)
         temp.append(mpItem)
-        competitiveNoInventory.append(temp)
+        cantCompete.append(temp)
-        return
-    temp=[]
-    temp.append(flipkartDetails)
-    temp.append(val)
-    flipkartPricing = __FlipkartPricing(ourSp,ourTp,lowestTp,lowestPossibleTp,None,lowestPossibleSp,prefSellerTp)
-    temp.append(flipkartPricing)
-    temp.append(mpItem)
-    cantCompete.append(temp)
-    #return cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap
+    return cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap
 def getOtherTp(flipkartDetails,val,spm,prefferedSeller):
     if val.parent_category==10011 or val.parent_category==12001:
         commissionPercentage = spm.competitorCommissionAccessory
     else:
             smtpServer.sendmail(sender, recipients, msg.as_string())
             print "Successfully sent email"
         except:
             print "Error: unable to send email."
+def populateScrapingResults(val):
+    global testcount
-def test(start,end):
+    testcount+=1
-    print "Start "+str(start)+" end"+str(end)
+    print "*****testcount",testcount
+    print val
+    try:
+        flipkartDetails = fetchDetails(val.fkSerialNumber,scraper)
+        val.flipkartDetails = flipkartDetails
+    except Exception as e:
+        print "Unable to fetch details of",val.fkSerialNumber
+        print e
+        val.flipkartDetails = None
+        return
+    try:
+        request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(val.skuAtFlipkart))
+        r = requests.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
+        print "Inventory info",r.json()
+        stock_count = int((r.json()['attributeValues'])['stock_count'])
+    except:
+        stock_count = 0
+    val.ourFlipkartInventory = stock_count
 def threadsToSpawn(runType,itemInfo,scraper,itemPopulated):
-    global threads
     if runType == RunType.FAVOURITE:
         startOffset = itemPopulated
         endOffset = startOffset
         while(len(threads)< 3 and endOffset<len(itemInfo)):
             endOffset = startOffset + 20
             if (endOffset >= len(itemInfo)):
                 endOffset = len(itemInfo)
-            t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset]))
+            t = threading.Thread(target=partial(populateScrapingResults, itemInfo[startOffset:endOffset],scraper))
             #t = threading.Thread(target=partial(test, startOffset, endOffset))
             threads.append(t)
             startOffset = startOffset + 20
         [t.start() for t in threads]
         [t.join() for t in threads]
         threads = []
         return endOffset
     else:
         count = 0
-        pool = ThreadPool(10)
+        pool = ThreadPool(5)
         startOffset = itemPopulated
         endOffset = startOffset
-        while(count<10 and endOffset<len(itemInfo)):
+        while(count<5 and endOffset<len(itemInfo)):
+            print "inside while",testcount
             endOffset = startOffset + 50
             if (endOffset >= len(itemInfo)):
                 endOffset = len(itemInfo)
             print itemInfo[startOffset:endOffset]
-            pool.map(decideCategory,itemInfo[startOffset:endOffset])
+            pool.map(populateScrapingResults,itemInfo[startOffset:endOffset])
             #t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
             #t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
             #t = threading.Thread(target=partial(test, startOffset, endOffset))
             #threads.append(t)
             startOffset = startOffset + 50
         #[t.start() for t in threads]
         #[t.join() for t in threads]
         #threads = []
         pool.close()
         pool.join()
+        "returning offset******"
+        time.sleep(5)
         return endOffset
 def main():
     parser = optparse.OptionParser()
     itemInfo= populateStuff(options.runType,timestamp)
     itemsPopulated = 0
     while (itemsPopulated<len(itemInfo)):
         itemsPopulated = threadsToSpawn(options.runType,itemInfo,scraper,itemsPopulated)
     previousProcessingTimestamp = session.query(func.max(MarketPlaceHistory.timestamp)).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).one()
+    cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = decideCategory(itemInfo)
     commitExceptionList(exceptionItems,timestamp)
     commitCantCompete(cantCompete,timestamp)
     commitBuyBox(buyBoxItems,timestamp)
     commitCompetitive(competitive,timestamp)
     commitCompetitiveNoInventory(competitiveNoInventory,timestamp)

Subversion Repositories SmartDukaan

(root)//trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py – Rev 11563 → 11571