| 13566 |
amit.gupta |
1 |
"""
|
|
|
2 |
soupselect.py
|
|
|
3 |
|
|
|
4 |
CSS selector support for BeautifulSoup.
|
|
|
5 |
|
|
|
6 |
soup = BeautifulSoup('<html>...')
|
|
|
7 |
select(soup, 'div')
|
|
|
8 |
- returns a list of div elements
|
|
|
9 |
|
|
|
10 |
select(soup, 'div#main ul a')
|
|
|
11 |
- returns a list of links inside a ul inside div#main
|
|
|
12 |
|
|
|
13 |
"""
|
|
|
14 |
|
|
|
15 |
import re
|
|
|
16 |
|
|
|
17 |
tag_re = re.compile('^[a-z0-9]+$')
|
|
|
18 |
|
|
|
19 |
attribselect_re = re.compile(
|
|
|
20 |
r'^(?P<tag>\w+)?\[(?P<attribute>\w+)(?P<operator>[=~\|\^\$\*]?)' +
|
|
|
21 |
r'=?"?(?P<value>[^\]"]*)"?\]$'
|
|
|
22 |
)
|
|
|
23 |
|
|
|
24 |
# /^(\w+)\[(\w+)([=~\|\^\$\*]?)=?"?([^\]"]*)"?\]$/
|
|
|
25 |
# \---/ \---/\-------------/ \-------/
|
|
|
26 |
# | | | |
|
|
|
27 |
# | | | The value
|
|
|
28 |
# | | ~,|,^,$,* or =
|
|
|
29 |
# | Attribute
|
|
|
30 |
# Tag
|
|
|
31 |
|
|
|
32 |
def attribute_checker(operator, attribute, value=''):
|
|
|
33 |
"""
|
|
|
34 |
Takes an operator, attribute and optional value; returns a function that
|
|
|
35 |
will return True for elements that match that combination.
|
|
|
36 |
"""
|
|
|
37 |
return {
|
|
|
38 |
'=': lambda el: el.get(attribute) == value,
|
|
|
39 |
# attribute includes value as one of a set of space separated tokens
|
|
|
40 |
'~': lambda el: value in el.get(attribute, '').split(),
|
|
|
41 |
# attribute starts with value
|
|
|
42 |
'^': lambda el: el.get(attribute, '').startswith(value),
|
|
|
43 |
# attribute ends with value
|
|
|
44 |
'$': lambda el: el.get(attribute, '').endswith(value),
|
|
|
45 |
# attribute contains value
|
|
|
46 |
'*': lambda el: value in el.get(attribute, ''),
|
|
|
47 |
# attribute is either exactly value or starts with value-
|
|
|
48 |
'|': lambda el: el.get(attribute, '') == value \
|
|
|
49 |
or el.get(attribute, '').startswith('%s-' % value),
|
|
|
50 |
}.get(operator, lambda el: el.has_key(attribute))
|
|
|
51 |
|
|
|
52 |
|
|
|
53 |
def select(soup, selector):
|
|
|
54 |
"""
|
|
|
55 |
soup should be a BeautifulSoup instance; selector is a CSS selector
|
|
|
56 |
specifying the elements you want to retrieve.
|
|
|
57 |
"""
|
|
|
58 |
tokens = selector.split()
|
|
|
59 |
current_context = [soup]
|
|
|
60 |
for token in tokens:
|
|
|
61 |
m = attribselect_re.match(token)
|
|
|
62 |
if m:
|
|
|
63 |
# Attribute selector
|
|
|
64 |
tag, attribute, operator, value = m.groups()
|
|
|
65 |
if not tag:
|
|
|
66 |
tag = True
|
|
|
67 |
checker = attribute_checker(operator, attribute, value)
|
|
|
68 |
found = []
|
|
|
69 |
for context in current_context:
|
|
|
70 |
found.extend([el for el in context.findAll(tag) if checker(el)])
|
|
|
71 |
current_context = found
|
|
|
72 |
continue
|
|
|
73 |
if '#' in token:
|
|
|
74 |
# ID selector
|
|
|
75 |
tag, id = token.split('#', 1)
|
|
|
76 |
if not tag:
|
|
|
77 |
tag = True
|
|
|
78 |
el = current_context[0].find(tag, {'id': id})
|
|
|
79 |
if not el:
|
|
|
80 |
return [] # No match
|
|
|
81 |
current_context = [el]
|
|
|
82 |
continue
|
|
|
83 |
if '.' in token:
|
|
|
84 |
# Class selector
|
|
|
85 |
tag, klass = token.split('.', 1)
|
|
|
86 |
if not tag:
|
|
|
87 |
tag = True
|
|
|
88 |
found = []
|
|
|
89 |
for context in current_context:
|
|
|
90 |
found.extend(
|
|
|
91 |
context.findAll(tag,
|
|
|
92 |
{'class': lambda attr: attr and klass in attr.split()}
|
|
|
93 |
)
|
|
|
94 |
)
|
|
|
95 |
current_context = found
|
|
|
96 |
continue
|
|
|
97 |
if token == '*':
|
|
|
98 |
# Star selector
|
|
|
99 |
found = []
|
|
|
100 |
for context in current_context:
|
|
|
101 |
found.extend(context.findAll(True))
|
|
|
102 |
current_context = found
|
|
|
103 |
continue
|
|
|
104 |
# Here we should just have a regular tag
|
|
|
105 |
if not tag_re.match(token):
|
|
|
106 |
return []
|
|
|
107 |
found = []
|
|
|
108 |
for context in current_context:
|
|
|
109 |
found.extend(context.findAll(token))
|
|
|
110 |
current_context = found
|
|
|
111 |
return current_context
|
|
|
112 |
|
|
|
113 |
def monkeypatch(BeautifulSoupClass=None):
|
|
|
114 |
"""
|
|
|
115 |
If you don't explicitly state the class to patch, defaults to the most
|
|
|
116 |
common import location for BeautifulSoup.
|
|
|
117 |
"""
|
|
|
118 |
if not BeautifulSoupClass:
|
|
|
119 |
from BeautifulSoup import BeautifulSoup as BeautifulSoupClass
|
|
|
120 |
BeautifulSoupClass.findSelect = select
|
|
|
121 |
|
|
|
122 |
def unmonkeypatch(BeautifulSoupClass=None):
|
|
|
123 |
if not BeautifulSoupClass:
|
|
|
124 |
from BeautifulSoup import BeautifulSoup as BeautifulSoupClass
|
|
|
125 |
delattr(BeautifulSoupClass, 'findSelect')
|