| 13569 |
amit.gupta |
1 |
'''
|
|
|
2 |
Created on Jan 15, 2015
|
|
|
3 |
|
|
|
4 |
@author: amit
|
|
|
5 |
'''
|
|
|
6 |
from pymongo.mongo_client import MongoClient
|
|
|
7 |
import importlib
|
|
|
8 |
import mechanize
|
| 13582 |
amit.gupta |
9 |
sourceMap = {1:"amazon", 2:"flipkart", 3:"snapdeal", 4:"spice", 5:"homeshop18"}
|
| 13569 |
amit.gupta |
10 |
|
| 13582 |
amit.gupta |
11 |
|
| 13569 |
amit.gupta |
12 |
def getStore(source_id):
|
|
|
13 |
#module = sourceMap[source_id]
|
|
|
14 |
store = Store(source_id)
|
|
|
15 |
module = importlib.import_module("dtr.sources." + sourceMap[source_id])
|
|
|
16 |
store = getattr(module, "Store")(source_id)
|
|
|
17 |
return store
|
|
|
18 |
|
|
|
19 |
class ScrapeException(Exception):
|
|
|
20 |
"""Exception raised for errors in the input.
|
|
|
21 |
|
|
|
22 |
Attributes:
|
|
|
23 |
expr -- input expression in which the error occurred
|
|
|
24 |
msg -- explanation of the error
|
|
|
25 |
"""
|
|
|
26 |
|
|
|
27 |
def __init__(self, expr, msg):
|
|
|
28 |
self.expr = expr
|
|
|
29 |
self.msg = msg
|
|
|
30 |
|
|
|
31 |
class ParseException(Exception):
|
|
|
32 |
"""Exception raised for errors in the input.
|
|
|
33 |
|
|
|
34 |
Attributes:
|
|
|
35 |
expr -- input expression in which the error occurred
|
|
|
36 |
msg -- explanation of the error
|
|
|
37 |
"""
|
|
|
38 |
|
|
|
39 |
def __init__(self, expr, msg):
|
|
|
40 |
self.expr = expr
|
|
|
41 |
self.msg = msg
|
|
|
42 |
|
|
|
43 |
class Store(object):
|
|
|
44 |
|
|
|
45 |
ORDER_PLACED = 'Order Placed'
|
|
|
46 |
ORDER_DELIVERED = 'Delivered'
|
|
|
47 |
ORDER_SHIPPED = 'Shipped' #Lets see if we can make use of it
|
|
|
48 |
ORDER_CANCELLED = 'Cancelled'
|
|
|
49 |
|
|
|
50 |
|
|
|
51 |
def __init__(self, store_id):
|
|
|
52 |
self.store_id = store_id
|
| 13576 |
amit.gupta |
53 |
self.store_name = sourceMap[store_id]
|
| 13569 |
amit.gupta |
54 |
|
|
|
55 |
|
|
|
56 |
def getName(self):
|
|
|
57 |
raise NotImplementedError
|
|
|
58 |
|
|
|
59 |
def scrapeAffiliate(self, startDate=None, endDate=None):
|
|
|
60 |
raise NotImplementedError
|
|
|
61 |
|
|
|
62 |
def saveToAffiliate(self, offers):
|
|
|
63 |
raise NotImplementedError
|
|
|
64 |
|
|
|
65 |
def scrapeStoreOrders(self,):
|
|
|
66 |
raise NotImplementedError
|
|
|
67 |
'''
|
|
|
68 |
Parses the order for specific store
|
|
|
69 |
|
|
|
70 |
order id, total amount, created on(now() if could not parse
|
|
|
71 |
suborder id, title, quantity, unit price, expected delivery date,
|
|
|
72 |
status (default would be Order placed)
|
|
|
73 |
|
|
|
74 |
once products are identified, each suborder can then be updated
|
|
|
75 |
with respective cashback.
|
|
|
76 |
|
|
|
77 |
Possible fields to display for Not yet delivered orders are
|
|
|
78 |
Product/Quantity/Amount/Store/CashbackAmount/OrderDate/ExpectedDelivery/OrderStaus/DetailedStatus/CashbackStatus
|
|
|
79 |
No need to show cancelled orders.
|
|
|
80 |
CashbackStatus - Pending/Approved/Cancelled/CreditedToWallet
|
|
|
81 |
OrderStatus - Placed/Cancelled/Delivered
|
|
|
82 |
'''
|
| 13576 |
amit.gupta |
83 |
def parseOrderRawHtml(self, orderId, subTagId, userId, rawHtml, orderSuccessUrl):
|
| 13569 |
amit.gupta |
84 |
|
|
|
85 |
pass
|
|
|
86 |
|
|
|
87 |
if __name__ == '__main__':
|
|
|
88 |
store = getStore(3)
|
|
|
89 |
store.scrapeAffiliate()
|
|
|
90 |
|
|
|
91 |
|
|
|
92 |
def getBrowserObject():
|
|
|
93 |
import cookielib
|
|
|
94 |
br = mechanize.Browser(factory=mechanize.RobustFactory())
|
|
|
95 |
cj = cookielib.LWPCookieJar()
|
|
|
96 |
br.set_cookiejar(cj)
|
|
|
97 |
br.set_handle_equiv(True)
|
|
|
98 |
br.set_handle_redirect(True)
|
|
|
99 |
br.set_handle_referer(True)
|
|
|
100 |
br.set_handle_robots(False)
|
|
|
101 |
br.set_debug_http(False)
|
|
|
102 |
br.set_debug_redirects(False)
|
|
|
103 |
br.set_debug_responses(False)
|
|
|
104 |
|
|
|
105 |
br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
|
|
|
106 |
|
|
|
107 |
br.addheaders = [('User-agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'),
|
|
|
108 |
('Accept', 'text/html,application/xhtml+xml,application/json,application/xml;q=0.9,*/*;q=0.8'),
|
|
|
109 |
('Accept-Encoding', 'gzip,deflate,sdch'),
|
|
|
110 |
('Accept-Language', 'en-US,en;q=0.8'),
|
|
|
111 |
('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3')]
|
|
|
112 |
return br
|