Subversion Repositories SmartDukaan

Rev

Rev 13845 | Rev 13876 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13828 kshitij.so 1 
import pymongo
2 
from dtr.utils.utils import to_java_date
3 
from datetime import datetime
4 
from operator import itemgetter
13833 kshitij.so 5 
from dtr.utils import FlipkartScraper
13828 kshitij.so 6 
 
7 
con = None
8 
now = datetime.now()
9 
print to_java_date(now)
10 
scraperFk = FlipkartScraper.FlipkartScraper()
11 
 
12 
def get_mongo_connection(host='localhost', port=27017):
13 
    global con
14 
    if con is None:
15 
        print "Establishing connection %s host and port %d" %(host,port)
16 
        try:
17 
            con = pymongo.MongoClient(host, port)
18 
        except Exception, e:
19 
            print e
20 
            return None
21 
    return con
22 
 
23 
def scrapeFlipkart():
24 
    flipkartBestSellers = list(get_mongo_connection().Catalog.MasterData.find({'rank':{'$gt':0},'source_id':2}))
25 
    for data in flipkartBestSellers:
26 
        inStock = 0
13841 kshitij.so 27 
        retryCount = 0
13828 kshitij.so 28 
        print str(data['identifier'])
13845 kshitij.so 29 
        if data['identifier'] is None or len(data['identifier'].strip())==0:
13828 kshitij.so 30 
            print "continue"
31 
            continue
32 
 
33 
        url = "http://www.flipkart.com/ps/%s"%(data['identifier'])
13841 kshitij.so 34 
        while(retryCount < 3):
35 
            try:
36 
                vendorsData = scraperFk.read(url)
37 
                fetched = True
13848 kshitij.so 38 
                break
13841 kshitij.so 39 
            except Exception as e:
13842 kshitij.so 40 
                print "***Retry count ",retryCount
13841 kshitij.so 41 
                retryCount+=1
42 
                if retryCount == 3:
43 
                    fetched = False
44 
                print e
45 
        if not fetched:
46 
            print "Unable to fetch data after multiple tries.Continue for ",data['identifier']
47 
            continue
13828 kshitij.so 48 
        sortedVendorsData = []
49 
        sortedVendorsData = sorted(vendorsData, key=itemgetter('sellingPrice'))
50 
        print "data",sortedVendorsData
51 
        lowestSp, iterator = (0,)*2
52 
        for vData in sortedVendorsData:
53 
            if iterator == 0:
54 
                lowestSp = vData['sellingPrice']
55 
            break
56 
        if lowestSp > 0:
57 
            inStock = 1
58 
        print lowestSp
59 
        print inStock
60 
        if lowestSp > 0:
61 
            get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier']}, {'$set' : {'available_price':lowestSp,'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
62 
        else:
63 
            get_mongo_connection().Catalog.MasterData.update({'identifier':data['identifier']}, {'$set' : {'updatedOn':to_java_date(now),'in_stock':inStock}}, multi=True)
64 
        print "+++++++++++++++++++++++++++"
65 
 
66 
 
67 
 
68 
 
69 
def main():
70 
    scrapeFlipkart()
71 
 
72 
if __name__=='__main__':
73 
    main()