Subversion Repositories SmartDukaan

Rev

Rev 12411 | Rev 12430 | Go to most recent revision | Show entire file | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed

Rev 12411 Rev 12412
Line 52... Line 52...
52 
            x.close()
 52 
            x.close()
53 
            sellerCount=0
 53 
            sellerCount=0
54 
            info = []
 54 
            info = []
55 
            sellerData = soup.findAll("div" , {"class" : "a-row a-spacing-mini olpOffer"})
 55 
            sellerData = soup.findAll("div" , {"class" : "a-row a-spacing-mini olpOffer"})
56 
            dataLength = len(sellerData)
 56 
            dataLength = len(sellerData)
57 
            print dataLength
 - 
 
58 
            for data in sellerData:
 57 
            for data in sellerData:
59 
                tempMap={}
 58 
                tempMap={}
60 
                price = data.find('span', attrs={'class' : re.compile('.*olpOfferPrice*')}).find('span').text
 59 
                price = data.find('span', attrs={'class' : re.compile('.*olpOfferPrice*')}).find('span').text
61 
                unitCost = float(price.replace("Rs.","").replace(",",""))
 60 
                unitCost = float(price.replace("Rs.","").replace(",",""))
62 
                shippingCost = data.find('p', attrs={'class' : re.compile('.*olpShippingInfo*')}).find('span').text
 61 
                shippingCost = data.find('p', attrs={'class' : re.compile('.*olpShippingInfo*')}).find('span').text
Line 72... Line 71...
72 
                if findStore:
 71 
                if findStore:
73 
                    storeUrl = sellerColumn.find('a')['href']
 72 
                    storeUrl = sellerColumn.find('a')['href']
74 
                    temp =  sellerColumn.find('a')
 73 
                    temp =  sellerColumn.find('a')
75 
                    store = temp.text
 74 
                    store = temp.text
76 
                    if len(store)==0:
 75 
                    if len(store)==0:
77 
                        print storeUrl
 - 
 
78 
                        dom_in = storeUrl.find("www.amazon.in")
 76 
                        dom_in = storeUrl.find("www.amazon.in")
79 
                        print dom_in
 - 
 
80 
                        if dom_in ==-1:
 77 
                        if dom_in ==-1:
81 
                            storeUrl="http://amazon.in"+storeUrl
 78 
                            storeUrl="http://amazon.in"+storeUrl
82 
                        if storeUrl[storeUrl.rfind('/')+1:]=='AF6E3O0VE0X4D':
 79 
                        if storeUrl[storeUrl.rfind('/')+1:]=='AF6E3O0VE0X4D':
83 
                            store = 'Saholic'
 80 
                            store = 'Saholic'
84 
                    if len(store)!=0:
 81 
                    if len(store)!=0:
Line 109... Line 106...
109 
            return returnMap
 106 
            return returnMap
110
107
111 
    def findStoreFront(self,returnMap):
 108 
    def findStoreFront(self,returnMap):
112 
        storeFront={}
 109 
        storeFront={}
113 
        for arr in returnMap.itervalues():
 110 
        for arr in returnMap.itervalues():
114 
            print "arr is ",arr
 - 
 
115 
            for dic in arr:
 111 
            for dic in arr:
116 
                print "dic ",dic
 - 
 
117 
                if dic['isStoreFront']!='True':
 112 
                if dic['isStoreFront']!='True':
118 
                    storeFront[dic.get('storeUrl')] =''
 113 
                    storeFront[dic.get('storeUrl')] =''
119 
        rs = (grequests.get(u,stream=False) for u in storeFront.keys())
 114 
        rs = (grequests.get(u,stream=False) for u in storeFront.keys())
120 
        for x in grequests.map(rs):
 115 
        for x in grequests.map(rs):
121 
            soup = strip_tags(x.text,invalid_tags)
 116 
            soup = strip_tags(x.text,invalid_tags)