Subversion Repositories SmartDukaan

Rev

Rev 13658 | Rev 13690 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed

Rev Author Line No. Line
13569 amit.gupta 1 
'''
2 
Created on Jan 15, 2015
3 
 
4 
@author: amit
5 
'''
6 
from dtr import main
13658 manas 7 
import importlib
8 
from dtr.main import getBrowserObject, getStore, ParseException
9 
from dtr.main import sourceMap
10 
from pymongo.mongo_client import MongoClient
11 
from BeautifulSoup import BeautifulSoup
12 
import re
13 
from dtr.dao import AffiliateInfo, Order, SubOrder
14 
import urllib
15 
 
16 
from pprint import pprint
17 
USERNAME='saholic1@gmail.com'
18 
PASSWORD='spice@2020'
19 
ORDER_TRACK_URL='https://m.flipkart.com/order_details'
13679 manas 20 
AFFILIATE_URL='https://www.flipkart.com/affiliate/login'
13658 manas 21 
 
13679 manas 22 
 
13569 amit.gupta 23 
class Store(main.Store):
13658 manas 24 
    OrderStatusMap = {
25 
                      main.Store.ORDER_PLACED : ['In Progress','N/A'],
26 
                      main.Store.ORDER_DELIVERED : ['Delivered'],
27 
                      main.Store.ORDER_SHIPPED : ['In Transit'],
28 
                      main.Store.ORDER_CANCELLED : ['Closed For Vendor Reallocation']
29 
 
30 
                      }
31 
    def __init__(self,store_id):
32 
        client = MongoClient('mongodb://localhost:27017/')
33 
        self.db = client.dtr
34 
        super(Store, self).__init__(store_id)
35 
 
13569 amit.gupta 36 
    def getName(self):
37 
        return "flipkart"
38 
 
13679 manas 39 
    def scrapeAffiliate(self, startDate=None, endDate=None):
40 
        br = getBrowserObject()
41 
        br.open(AFFILIATE_URL)
42 
        print br.select_form(name="fk-signin")
43 
 
13569 amit.gupta 44 
 
13658 manas 45 
 
46 
    def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
47 
        br = getBrowserObject()
48 
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
49 
 
50 
        response = br.open(url)
51 
        page = ungzipResponse(response, br)
52 
 
53 
 
54 
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
55 
 
56 
        sections = soup.findAll("div", {"class":"ui-app-card-body"})
57 
        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})
58 
        sections.pop(1)
59 
 
60 
        #fetching summary of the products
61 
        for data in sections:
62 
            name = data.findAll("span")
63 
            i=0
64 
            print "Summary Details of Flipkart"
65 
            while i< len(name):
66 
                if "key" in str(name[i]):
67 
                    if "Grand Total" in str(name[i]):
68 
                        #Total Amount
69 
                        print "Total Amount " + name[i+1].text
70 
                    elif "Status" in str(name[i]):
71 
                        print "Summary Status " + name[i+1].text
72 
                    elif "Order ID" in str(name[i]):
73 
                        j=i+1
74 
                        while j < i+5:
75 
                            if "value" in str(name[j]):
76 
                                print "Order ID " + str(j-i) + " " +name[j].text
77 
                                j=j+1
78 
                            else:
79 
                                break
80 
                    elif "Order Date" in str(name[i]):
81 
                        print "Order Date " + name[i+1].text
82 
                i=i+1
83 
        print "Details of Sub Orders"
84 
        #fetching suborders details
85 
        for subOrder in mainOrder:
86 
            subOrd = subOrder.findAll("li")
87 
 
88 
            subOrd.pop(len(subOrd)-1)
89 
            for productInfo in subOrd:
90 
                y = productInfo.findAll("span")
91 
                for suborderId in y:
92 
                    if "value emp" in str(suborderId):
93 
                        #Sub Order ID
94 
                        print "Sub Order ID"  + suborderId.text
13679 manas 95 
                g = productInfo.findAll("div",{"class":"product-image-block"})
96 
                for imageUrl in g:
97 
                    print str(imageUrl.find("img")['src'])
13658 manas 98 
                x = productInfo.findAll("div",{"class":"product-info"})
99 
                for maik in x:
100 
                    #Product title
101 
                    print maik.find("a",{"class":"product-title"}).text
13679 manas 102 
                    print str(maik.find("a")['href'])
103 
                    start='pid='
104 
                    s=str(maik.find("a")['href'])
105 
                    print re.findall(re.escape(start)+"(.*)",s)[0]
13658 manas 106 
                    mname = maik.findAll("span")
107 
                    k=0
108 
                    while k<len(mname):
109 
                        if "note" in str(mname[k]):
110 
                            if "Color:" in str(mname[k]):
111 
                                #Color
112 
                                print "Color " + mname[k+1].text
113 
                            elif "Qty:" in str(mname[k]):
114 
                                #Quantity of sub item
115 
                                print "Quantity "+mname[k+1].text
116 
                            elif "Subtotal:" in str(mname[k]):
117 
                                #SubTotal for the item
118 
                                print "Sub Total"+mname[k+1].text
119 
                            elif "Delivery:" in str(mname[k]):
120 
                                #Delivery Date for sub order
13679 manas 121 
                                print "Delivery Date " +mname[k+1].text
122 
                        if "key text-dim" in str(mname[k]):
123 
                            if "Seller" in str(mname[k]):
124 
                                print "Seller  URL : http://m.flipkart.com"+maik.findAll("a")[1]['href']
125 
                                print "Seller "+maik.findAll("a")[1].text
13658 manas 126 
                        k=k+1
127 
                status=-1
128 
                #To track shipping details
129 
                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})
130 
                for trackingStatus in n:
131 
                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
132 
                    if "approveDetails-complete" in str(tr):
133 
                        if "processingDetails-complete" in str(tr):
134 
                            if "shippingDetails-complete" in str(tr):
135 
                                if "delivery-complete" in str(tr):
136 
                                    status = 3
137 
                                else:
138 
                                    status = 2
139 
                            else:
140 
                                status = 1
141 
                        else:
13679 manas 142 
                            status = 0
143 
                    elif "approveDetails-ongoing" in str(tr):
144 
                        status=0
13658 manas 145 
                    if "dead" in str(tr):
146 
                        status = 4
147 
                print "Sub Order Status " + str(status)
148 
                f = productInfo.findAll("div",{"class":"c-tabs-content m-top active"})
149 
                for trackingDetailsActive in f:
150 
                    print "Sub order tracking description " + trackingDetailsActive.text
151 
 
152 
 
153 
    def flipkartOrderTracking(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
154 
        br = getBrowserObject()
155 
        url = ORDER_TRACK_URL + re.findall('.*(\?.*?)$', orderSuccessUrl,re.IGNORECASE)[0]
156 
 
157 
        response = br.open(url)
158 
        page = ungzipResponse(response, br)
159 
 
160 
 
161 
        soup = BeautifulSoup(page,convertEntities=BeautifulSoup.HTML_ENTITIES)
162 
 
163 
 
164 
        mainOrder = soup.findAll("ul",{"class":"m-bottom p-cart"})
165 
        #fetching suborders details
166 
        for subOrder in mainOrder:
167 
            subOrd = subOrder.findAll("li")
168 
 
169 
            subOrd.pop(len(subOrd)-1)
170 
            for productInfo in subOrd:
171 
                status=-1
172 
                #To track shipping details
173 
                n = productInfo.findAll("div",{"class":"tracking-progress grid-row cf"})
174 
                for trackingStatus in n:
175 
                    tr = trackingStatus.findAll("div",{"class":"tap-bullet-area c-tab-trigger"})
176 
                    if "approveDetails-complete" in str(tr):
13679 manas 177 
                        print "Here"
13658 manas 178 
                        if "processingDetails-complete" in str(tr):
13679 manas 179 
                            print "There"
13658 manas 180 
                            if "shippingDetails-complete" in str(tr):
13679 manas 181 
                                print "Share"
13658 manas 182 
                                if "delivery-complete" in str(tr):
13679 manas 183 
                                    print "Last"
13658 manas 184 
                                    status = 3
185 
                                else:
186 
                                    status = 2
187 
                            else:
188 
                                status = 1
189 
                        else:
190 
                            status = 0
191 
                    if "dead" in str(tr):
192 
                        status = 4
193 
                print status
194 
 
195 
 
196 
 
197 
def _saveToOrderFlipkart(self, order):
198 
        collection = self.db.merchantOrder
199 
        order = collection.insert(order)
200 
 
201 
def main():
202 
 
203 
    store = getStore(2)
13679 manas 204 
    store.scrapeAffiliate()
13658 manas 205 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3016502908102575&token=0db4c692bacbfbfc158b52358ac9e91e&src=or&pr=1')
206 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD3018701137253850&token=f7402ddcf2b63b37cc6bc528cc115d2f&src=or&pr=1')
13679 manas 207 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
13658 manas 208 
    #store.flipkartOrderTracking(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019279584515727&token=7d85d8c24d36b5a1efc8008634390c7e&src=or&pr=1')
13679 manas 209 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://m.flipkart.com/orderresponse?reference_id=OD0019365336126533&token=dbce2bd4dc4023295b436a7d3c7986c9&src=or&pr=1')
210 
    #store.parseOrderRawHtml(12346, "subtagId", 122324,  "html", 'https://dl.flipkart.com/orderresponse?reference_id=OD1019453634552336&token=e8e04871ad65b532aa53fa82bb34b901&src=or&pr=1')
211 
 
13658 manas 212 
def ungzipResponse(r,b):
213 
        headers = r.info()
214 
        if headers['Content-Encoding']=='gzip':
215 
            import gzip
216 
        print "********************"
217 
        print "Deflating gzip response"
218 
        print "********************"
219 
        gz = gzip.GzipFile(fileobj=r, mode='rb')
220 
        html = gz.read()
221 
        gz.close()
222 
        return html
223 
 
224 
if __name__ == '__main__':
225 
        main()
226 
 
227 
def todict(obj, classkey=None):
228 
    if isinstance(obj, dict):
229 
        data = {}
230 
        for (k, v) in obj.items():
231 
            data[k] = todict(v, classkey)
232 
        return data
233 
    elif hasattr(obj, "_ast"):
234 
        return todict(obj._ast())
235 
    elif hasattr(obj, "__iter__"):
236 
        return [todict(v, classkey) for v in obj]
237 
    elif hasattr(obj, "__dict__"):
238 
        data = dict([(key, todict(value, classkey))
239 
            for key, value in obj.__dict__.iteritems()
240 
            if not callable(value) and not key.startswith('_')])
241 
        if classkey is not None and hasattr(obj, "__class__"):
242 
            data[classkey] = obj.__class__.__name__
243 
        return data
244 
    else:
245 
        return obj