WebSVN – SmartDukaan – Diff – /trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py

 from datetime import date, datetime, timedelta
 from shop2020.utils import EmailAttachmentSender
 from shop2020.utils.EmailAttachmentSender import get_attachment_part
 import math
 from operator import itemgetter
+from functools import partial
 import simplejson as json
 import xlwt
 import optparse
 import sys
 import smtplib
+import threading
+from multiprocessing import Process
 from email.mime.text import MIMEText
 import email
 from email.mime.multipart import MIMEMultipart
 import email.encoders
 import cookielib
 syncPrice=config_client.get_property('sync_price_on_marketplace')
 DataService.initialize(db_hostname=host)
+threads = []
 inventoryMap = {}
 itemSaleMap = {}
+cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = [],[],[],[],[],[],[],[]
 class __FlipkartDetails:
     def __init__(self,rank ,ourSp , secondLowestSellerSp, prefSellerSp, lowestSellerSp, lowestSellerScore, prefSellerScore, secondLowestSellerScore, ourScore, shippingTimeLowerLimitLowestSeller,shippingTimeUpperLimitLowestSeller, \
     shippingTimeLowerLimitPrefSeller, shippingTimeUpperLimitPrefSeller, shippingTimeLowerLimitOur, shippingTimeUpperLimitOur, shippingTimeLowerLimitSecondLowestSeller, shippingTimeUpperLimitSecondLowestSeller, totalAvailableSeller, lowestSellerName, lowestSellerCode, secondLowestSellerName, secondLowestSellerCode, prefSellerName, prefSellerCode, lowestSellerBuyTrend, \
         self.secondLowestSellerBuyTrend = secondLowestSellerBuyTrend
         self.ourCode = ourCode
 class __FlipkartItemInfo:
-    def __init__(self, fkSerialNumber, nlc, courierCost, item_id, product_group, brand, model_name, model_number, color, weight, parent_category, risky, warehouseId, vatRate, runType, parent_category_name, sourcePercentage, ourFlipkartInventory):
+    def __init__(self, fkSerialNumber, nlc, courierCost, item_id, product_group, brand, model_name, model_number, color, weight, parent_category, risky, warehouseId, vatRate, runType, parent_category_name, sourcePercentage, ourFlipkartInventory, skuAtFlipkart):
         self.fkSerialNumber = fkSerialNumber
         self.nlc = nlc
         self.courierCost = courierCost
         self.item_id = item_id
         self.warehouseId = warehouseId
         self.vatRate = vatRate
         self.runType = runType
         self.parent_category_name = parent_category_name
         self.sourcePercentage = sourcePercentage
         self.ourFlipkartInventory = ourFlipkartInventory
+        self.skuAtFlipkart = skuAtFlipkart
 class __FlipkartPricing:
     def __init__(self, ourSp, ourTp, lowestTp, lowestPossibleTp, secondLowestSellerTp, lowestPossibleSp, prefSellerTp):
         self.ourTp = ourTp
             if scp is not None:
                 sourcePercentage = scp
             else:
                 spm = SourcePercentageMaster.get_by(source=OrderSource.FLIPKART)
                 sourcePercentage = spm
-        try:
+#        try:
-            request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(flipkart_item.skuAtFlipkart))
+#            request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(flipkart_item.skuAtFlipkart))
-            r = requests.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
+#            r = requests.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
-            print "Inventory info",r.json()
+#            print "Inventory info",r.json()
-            stock_count = int((r.json()['attributeValues'])['stock_count'])
+#            stock_count = int((r.json()['attributeValues'])['stock_count'])
-        except:
+#        except:
-            stock_count = 0
+#            stock_count = 0
-        flipkartItemInfo = __FlipkartItemInfo(flipkart_item.flipkartSerialNumber, flipkart_item.maxNlc,mp_item.courierCost, it.id, it.product_group, it.brand, it.model_name, it.model_number, it.color, it.weight, category.parent_category_id, it.risky, flipkart_item.warehouseId, None, runType, parent_category.display_name,sourcePercentage,stock_count)
+        flipkartItemInfo = __FlipkartItemInfo(flipkart_item.flipkartSerialNumber, flipkart_item.maxNlc,mp_item.courierCost, it.id, it.product_group, it.brand, it.model_name, it.model_number, it.color, it.weight, category.parent_category_id, it.risky, flipkart_item.warehouseId, None, runType, parent_category.display_name,sourcePercentage,None,flipkart_item.skuAtFlipkart)
         itemInfo.append(flipkartItemInfo)
     return itemInfo
 def fetchDetails(flipkartSerialNumber,scraper):
     url = "http://www.flipkart.com/ps/%s"%(flipkartSerialNumber)
         return 'MEDIUM'
     else:
         return 'LOW'
 def decideCategory(itemInfo,scraper):
+    print "length****",len(itemInfo)
     global itemSaleMap
+    global cantCompete
+    global buyBoxItems
+    global competitive
-    cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap = [],[],[],[],[],[],[],[]
+    global competitiveNoInventory
+    global exceptionItems
+    global negativeMargin
+    global cheapButNotPref
+    global prefButNotCheap
     catalog_client = CatalogClient().get_client()
     inventory_client = InventoryClient().get_client()
     for val in itemInfo:
         spm = val.sourcePercentage
             print "Unable to fetch details of",val.fkSerialNumber
             print e
             exceptionItems.append(val)
             continue
+        try:
+            request_url = "https://api.flipkart.net/sellers/skus/%s/listings"%(str(val.skuAtFlipkart))
+            r = requests.get(request_url, auth=('m2z93iskuj81qiid', '0c7ab6a5-98c0-4cdc-8be3-72c591e0add4'))
+            print "Inventory info",r.json()
+            stock_count = int((r.json()['attributeValues'])['stock_count'])
+        except:
+            stock_count = 0
+        val.ourFlipkartInventory = stock_count
         if (flipkartDetails.totalAvailableSeller==0):
             exceptionItems.append(val)
             continue
         mpItem = MarketplaceItems.get_by(itemId=val.item_id,source=OrderSource.FLIPKART)
         flipkartPricing = __FlipkartPricing(ourSp,ourTp,lowestTp,lowestPossibleTp,None,lowestPossibleSp,prefSellerTp)
         temp.append(flipkartPricing)
         temp.append(mpItem)
         cantCompete.append(temp)
-    return cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap
+    #return cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap
 def getOtherTp(flipkartDetails,val,spm,prefferedSeller):
     if val.parent_category==10011 or val.parent_category==12001:
         commissionPercentage = spm.competitorCommissionAccessory
     else:
         sheet_iterator+=1
     filename = "/tmp/flipkart-report-"+runType+" " + str(timestamp) + ".xls"
     wbk.save(filename)
     try:
-        #EmailAttachmentSender.mail("build@shop2020.in", "cafe@nes", ["rajneesh.arora@saholic.com","anikendra.das@saholic.com","kshitij.sood@saholic.com"], " Flipkart Auto Pricing "+runType+" " + str(timestamp), "", [get_attachment_part(filename)], [""], [])
+        EmailAttachmentSender.mail("build@shop2020.in", "cafe@nes", ["kshitij.sood@saholic.com"], " Flipkart Auto Pricing "+runType+" " + str(timestamp), "", [get_attachment_part(filename)], [""], [])
-        EmailAttachmentSender.mail("build@shop2020.in", "cafe@nes", ["chandan.kumar@saholic.com","manoj.kumar@saholic.com","yukti.jain@saholic.com","ankush.dhingra@saholic.com","manoj.pal@saholic.com"], " Flipkart Scraping "+runType+" " + str(timestamp), "", [get_attachment_part(filename)], ["rajneesh.arora@saholic.com","anikendra.das@saholic.com","vikram.raghav@saholic.com","kshitij.sood@saholic.com","chaitnaya.vats@saholic.com","khushal.bhatia@saholic.com"], [])
+        #EmailAttachmentSender.mail("build@shop2020.in", "cafe@nes", ["chandan.kumar@saholic.com","manoj.kumar@saholic.com","yukti.jain@saholic.com","ankush.dhingra@saholic.com","manoj.pal@saholic.com"], " Flipkart Scraping "+runType+" " + str(timestamp), "", [get_attachment_part(filename)], ["rajneesh.arora@saholic.com","anikendra.das@saholic.com","vikram.raghav@saholic.com","kshitij.sood@saholic.com","chaitnaya.vats@saholic.com","khushal.bhatia@saholic.com"], [])
     except Exception as e:
         print e
         print "Unable to send report.Trying with local SMTP"
         smtpServer = smtplib.SMTP('localhost')
         smtpServer.set_debuglevel(1)
         sender = 'support@shop2020.in'
-        #recipients = ["kshitij.sood@saholic.com"]
+        recipients = ["kshitij.sood@saholic.com"]
         msg = MIMEMultipart()
         msg['Subject'] = "Flipkart Scraping" + ' '+runType+' - ' + str(datetime.now())
         msg['From'] = sender
-        recipients = ['rajneesh.arora@saholic.com','anikendra.das@saholic.com','vikram.raghav@saholic.com','kshitij.sood@saholic.com','khushal.bhatia@saholic.com','chaitnaya.vats@saholic.com','chandan.kumar@saholic.com','manoj.kumar@saholic.com','yukti.jain@saholic.com','ankush.dhingra@saholic.com','manoj.pal@saholic.com']
+        #recipients = ['rajneesh.arora@saholic.com','anikendra.das@saholic.com','vikram.raghav@saholic.com','kshitij.sood@saholic.com','khushal.bhatia@saholic.com','chaitnaya.vats@saholic.com','chandan.kumar@saholic.com','manoj.kumar@saholic.com','yukti.jain@saholic.com','ankush.dhingra@saholic.com','manoj.pal@saholic.com']
         msg['To'] = ",".join(recipients)
         fileMsg = email.mime.base.MIMEBase('application','vnd.ms-excel')
         fileMsg.set_payload(file(filename).read())
         email.encoders.encode_base64(fileMsg)
         fileMsg.add_header('Content-Disposition','attachment;filename=flipkart.xls')
         try:
             smtpServer.sendmail(sender, recipients, msg.as_string())
             print "Successfully sent email"
         except:
             print "Error: unable to send email."
+def test(start,end):
+    print "Start "+str(start)+" end"+str(end)
+def threadsToSpawn(runType,itemInfo,scraper,itemPopulated):
+    global threads
+    if runType == RunType.FAVOURITE:
+        startOffset = itemPopulated
+        endOffset = startOffset
+        while(len(threads)< 3 and endOffset<len(itemInfo)):
+            endOffset = startOffset + 20
+            if (endOffset >= len(itemInfo)):
+                endOffset = len(itemInfo)
+            t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
+            #t = threading.Thread(target=partial(test, startOffset, endOffset))
+            threads.append(t)
+            startOffset = startOffset + 20
+        [t.start() for t in threads]
+        [t.join() for t in threads]
+        threads = []
+        return endOffset
+    else:
+        startOffset = itemPopulated
+        endOffset = startOffset
+        while(len(threads)< 10 and endOffset<len(itemInfo)):
+            endOffset = startOffset + 50
+            if (endOffset >= len(itemInfo)):
+                endOffset = len(itemInfo)
+            t = Process(target=decideCategory,args=(itemInfo[startOffset:endOffset], scraper))
+            #t = threading.Thread(target=partial(decideCategory, itemInfo[startOffset:endOffset], scraper))
+            #t = threading.Thread(target=partial(test, startOffset, endOffset))
+            threads.append(t)
+            startOffset = startOffset + 50
+        [t.start() for t in threads]
+        [t.join() for t in threads]
+        threads = []
+        return endOffset
 def main():
     parser = optparse.OptionParser()
     parser.add_option("-t", "--type", dest="runType",
                    default="FULL", type="string",
                    help="Run type FULL or FAVOURITE")
         print "Run type argument illegal."
         sys.exit(1)
     timestamp = datetime.now()
     scraper = FlipkartScraper.FlipkartScraper()
     itemInfo= populateStuff(options.runType,timestamp)
+    itemsPopulated = 0
-    cantCompete, buyBoxItems, competitive, \
+    while (itemsPopulated<len(itemInfo)):
-    competitiveNoInventory, exceptionList, negativeMargin, cheapButNotPref, prefButNotCheap = decideCategory(itemInfo,scraper)
+        itemsPopulated = threadsToSpawn(options.runType,itemInfo,scraper,itemsPopulated)
     previousProcessingTimestamp = session.query(func.max(MarketPlaceHistory.timestamp)).filter(MarketPlaceHistory.source==OrderSource.FLIPKART).one()
-    exceptionItems = commitExceptionList(exceptionList,timestamp)
+    commitExceptionList(exceptionItems,timestamp)
-    cantComepeteItems = commitCantCompete(cantCompete,timestamp)
+    commitCantCompete(cantCompete,timestamp)
-    buyBoxList = commitBuyBox(buyBoxItems,timestamp)
+    commitBuyBox(buyBoxItems,timestamp)
-    competitiveItems = commitCompetitive(competitive,timestamp)
+    commitCompetitive(competitive,timestamp)
-    competitiveNoInventoryItems = commitCompetitiveNoInventory(competitiveNoInventory,timestamp)
+    commitCompetitiveNoInventory(competitiveNoInventory,timestamp)
-    negativeMarginItems = commitNegativeMargin(negativeMargin,timestamp)
+    commitNegativeMargin(negativeMargin,timestamp)
-    cheapButNotPrefItems = commitCheapButNotPref(cheapButNotPref,timestamp)
+    commitCheapButNotPref(cheapButNotPref,timestamp)
-    prefButNotCheapItems = commitPrefButNotCheap(prefButNotCheap, timestamp)
+    commitPrefButNotCheap(prefButNotCheap, timestamp)
+    #getLastUpdatedOn(mailSender.start, mailSender.stop)
     #successfulAutoDecrease = fetchItemsForAutoDecrease(timestamp)
     #successfulAutoIncrease = fetchItemsForAutoIncrease(timestamp)
     if options.runType=='FULL':
         previousAutoFav, nowAutoFav = markAutoFavourite()
     if options.runType=='FULL':
-        write_report(cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionList, negativeMargin, cheapButNotPref, prefButNotCheap, previousAutoFav, nowAutoFav,timestamp, options.runType)
+        write_report(cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap, previousAutoFav, nowAutoFav,timestamp, options.runType)
     else:
-        write_report(cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionList, negativeMargin, cheapButNotPref, prefButNotCheap, None, None, timestamp, options.runType)
+        write_report(cantCompete, buyBoxItems, competitive, competitiveNoInventory, exceptionItems, negativeMargin, cheapButNotPref, prefButNotCheap, None, None, timestamp, options.runType)
 if __name__ == '__main__':
     main()

Subversion Repositories SmartDukaan

(root)/trunk/PyProj/src/shop2020/model/v1/catalog/script/FlipkartHistoryPopulator.py – Rev 11228 → 11560