Subversion Repositories SmartDukaan

Rev

Rev 13876 | Rev 13919 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 13876 Rev 13915
Line 1... Line 1...
1 
import pymongo
 1 
import pymongo
2 
from dtr.utils.utils import to_java_date
 2 
from dtr.utils.utils import to_java_date
3 
from datetime import datetime
 3 
from datetime import datetime, timedelta
4 
from operator import itemgetter
 4 
from operator import itemgetter
5 
from dtr.utils import FlipkartScraper
 5 
from dtr.utils import FlipkartScraper
6 
 
 6 
 
7 
con = None
 7 
con = None
8 
now = datetime.now()
 8 
now = datetime.now()
Line 28... Line 28...
28 
        print str(data['identifier'])
 28 
        print str(data['identifier'])
29 
        if data['identifier'] is None or len(data['identifier'].strip())==0:
 29 
        if data['identifier'] is None or len(data['identifier'].strip())==0:
30 
            print "continue"
 30 
            print "continue"
31 
            continue
 31 
            continue
32
32
- 
 
 33 
        try:
- 
 
 34 
            if data['updatedOn'] + timedelta(minutes=5) > to_java_date(now):
- 
 
 35 
                print "sku id is already updated",data['_id']
- 
 
 36 
                continue
- 
 
 37 
        except:
- 
 
 38 
            pass
- 
 
 39
33 
        url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
 40 
        url = "http://www.flipkart.com/ps/%s"%(data['identifier'].strip())
34 
        while(retryCount < 3):
 41 
        while(retryCount < 3):
35 
            try:
 42 
            try:
36 
                vendorsData = scraperFk.read(url)
 43 
                vendorsData = scraperFk.read(url)
37 
                fetched = True
 44 
                fetched = True
Line 43... Line 50...
43 
                    fetched = False
 50 
                    fetched = False
44 
                print e
 51 
                print e
45 
        if not fetched:
 52 
        if not fetched:
46 
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
 53 
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
47 
            continue
 54 
            continue
- 
 
 55
48 
        sortedVendorsData = []
 56 
        sortedVendorsData = []
49 
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
 57 
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
50 
        print "data",sortedVendorsData
 58 
        print "data",sortedVendorsData
51 
        lowestSp, iterator = (0,)*2
 59 
        lowestSp, iterator = (0,)*2
52 
        for vData in sortedVendorsData:
 60 
        for vData in sortedVendorsData:
Line 56... Line 64...
56 
        if lowestSp > 0:
 64 
        if lowestSp > 0:
57 
            inStock = 1
 65 
            inStock = 1
58 
        print lowestSp
 66 
        print lowestSp
59 
        print inStock
 67 
        print inStock
60 
        if lowestSp > 0:
 68 
        if lowestSp > 0:
61 
            get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier'].strip(),'source_id':2}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
 69 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
- 
 
 70 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'available_price':lowestSp , 'in_stock':inStock}}, multi=True)
62 
        else:
 71 
        else:
63 
            get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier'].strip(),'source_id':2}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
 72 
            get_mongo_connection().Catalog.MasterData.update({'_id':data['_id']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
64 
        print "+++++++++++++++++++++++++++"
 73 
            get_mongo_connection().Catalog.Deals.update({'_id':data['_id']}, {'$set' : {'in_stock':inStock}}, multi=True)
65
- 
 
66
74
67
75 
        try:
- 
 
 76 
            recomputeDeal(data['skuBundleId'])
- 
 
 77 
        except:
- 
 
 78 
            print "Unable to compute deal for ",data['skuBundleId']
- 
 
 79 
 
- 
 
 80 
def recomputeDeal(skuBundleId):
- 
 
 81 
    """Lets recompute deal for this bundle"""
- 
 
 82 
    print "Recomputing for bundleId",skuBundleId
- 
 
 83
- 
 
 84 
    similarItems = list(get_mongo_connection().Catalog.Deals.find({'skuBundleId':skuBundleId}).sort([('available_price',pymongo.ASCENDING)]))
- 
 
 85 
    bestPrice = float("inf")
- 
 
 86 
    bestOne = None
- 
 
 87 
    bestSellerPoints = 0
- 
 
 88 
    toUpdate = []
- 
 
 89 
    for similarItem in similarItems:
- 
 
 90 
        if similarItem['in_stock'] == 0:
- 
 
 91 
            get_mongo_connection().Catalog.Deals.update({ '_id' : similarItem['_id'] }, {'$set':{'showDeal':0 }})
- 
 
 92 
            continue
- 
 
 93 
        if similarItem['available_price'] < bestPrice:
- 
 
 94 
            bestOne = similarItem
- 
 
 95 
            bestPrice = similarItem['available_price']
- 
 
 96 
            bestSellerPoints = similarItem['bestSellerPoints']
- 
 
 97 
        elif similarItem['available_price'] == bestPrice and bestSellerPoints < similarItem['bestSellerPoints']:
- 
 
 98 
            bestOne = similarItem
- 
 
 99 
            bestPrice = similarItem['available_price']
- 
 
 100 
            bestSellerPoints = similarItem['bestSellerPoints']
- 
 
 101 
        else:
- 
 
 102 
            pass
- 
 
 103 
    if bestOne is not None:
- 
 
 104 
        for similarItem in similarItems:
- 
 
 105 
            toUpdate.append(similarItem['_id'])
- 
 
 106 
        toUpdate.remove(bestOne['_id'])
- 
 
 107 
        get_mongo_connection().Catalog.Deals.update({ '_id' : bestOne['_id'] }, {'$set':{'showDeal':1 }})
- 
 
 108 
    if len(toUpdate) > 0:
- 
 
 109 
        get_mongo_connection().Catalog.Deals.update({ '_id' : { "$in": toUpdate } }, {'$set':{'showDeal':0 }},upsert=False, multi=True)
68 
 
 110 
 
69 
def main():
 111 
def main():
70 
    scrapeFlipkart()
 112 
    scrapeFlipkart()
71
113
72 
if __name__=='__main__':
 114 
if __name__=='__main__':