annotate curlers.py @ 3:091c03f1b2e8

Getting it working...
author David Barts <n5jrn@me.com>
date Thu, 26 Dec 2019 19:54:45 -0800
parents
children 7a83e82e65a6
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
1 #!/usr/bin/env python3
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
3
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
4 # Classes for curling both HTML and plain text.
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
5
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
6 # I m p o r t s
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
7
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
8 import os, sys
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
9 from workspace import Workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
10
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
11 # V a r i a b l e s
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
12
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
13 # Quote types
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
14 LSQUO = "\u2018"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
15 APOS = RSQUO = "\u2019"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
16 LDQUO = "\u201C"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
17 RDQUO = "\u201D"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
18
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
19 # Words that start with an apostrophe. Cribbed from Wordpress.
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
20 _ASTART = [ "'tain't", "'twere", "'twas", "'tis", "'twill", "'til",
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
21 "'bout", "'nuff", "'round", "'cause" , "'em" ]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
22
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
23 # HTML tags that enclose raw data
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
24 _RAW = set(["script", "style"])
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
25
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
26 # HTML block elements
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
27 _BLOCK = set([
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
28 "address", "blockquote", "div", "dl", "fieldset", "form", "h1",
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
29 "h2", "h3", "h4", "h5", "h6", "hr", "noscript", "ol", "p", "pre",
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
30 "table", "ul"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
31 ])
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
32
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
33 # F u n c t i o n s
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
34
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
35 def _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
36 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
37 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
38 for i in _ASTART:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
39 li = len(i)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
40 if ws[pos:pos+li].lower() == i and not ws[pos+li].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
41 return True
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
42
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
43 # C l a s s e s
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
44
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
45 class BaseCurler():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
46 def feed(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
47 raise NotImplementedError()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
48
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
49 class TextCurler(BaseCurler):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
50 """
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
51 For processing plain text. Assumes the entire text is a block; it is
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
52 the responsibility of the caller to break the input into paragraphs.
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
53 """
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
54 def __init__(self, workspace):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
55 self.workspace = workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
56 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
57 self._pos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
58
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
59 def feed(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
60 self._pos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
61 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
62 for self._pos in range(len(self.workspace)):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
63 self._state()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
64
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
65 def _is_cockney(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
66 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
67 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
68 for i in _ASTART:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
69 li = len(i)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
70 print("comparing {0!r} and {1!r}\n".format(ws[pos:pos+li].lower(), i))
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
71 if ws[pos:pos+li].lower() == i and not ws[pos+li].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
72 return True
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
73
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
74 def _norm(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
75 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
76 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
77 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
78 if char == "\"":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
79 # opening double quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
80 ws[pos] = LDQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
81 self._state = self._seen_ld
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
82 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
83 # in this state, ' is always an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
84 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
85
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
86 def _seen_ld(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
87 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
88 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
89 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
90 if char == "\"":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
91 # closing double quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
92 ws[pos] = RDQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
93 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
94 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
95 if ws[pos-1].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
96 # either an inter-word, or an end of word, apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
97 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
98 elif ws[pos+1].isdecimal() or _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
99 # also an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
100 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
101 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
102 # opening single quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
103 ws[pos] = LSQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
104 self._state = self._seen_ls
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
105
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
106 def _seen_ls(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
107 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
108 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
109 if ws[pos] == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
110 if ws[pos-1].isalpha() and ws[pos+1].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
111 # obvious apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
112 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
113 elif ws[pos+1].isdecimal() or _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
114 # also an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
115 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
116 elif ws[pos-1].isspace():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
117 # start of word apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
118 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
119 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
120 # closing single quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
121 ws[pos] = RSQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
122 self._state = self._seen_ld
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
123
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
124 class HtmlCurler(BaseCurler):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
125 """
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
126 For processing HTML. Uses HTML block tags to delimit blocks.
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
127 """
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
128 def __init__(self, workspace):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
129 self.workspace = workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
130 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
131 self._pos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
132 self._ltpos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
133 self._endtag = None
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
134 self._ltstate = None
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
135
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
136 def feed(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
137 self._pos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
138 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
139 for self._pos in range(len(self.workspace)):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
140 self._state()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
141
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
142 def _is_cockney(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
143 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
144 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
145 for i in _ASTART:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
146 li = len(i)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
147 print("comparing {0!r} and {1!r}\n".format(ws[pos:pos+li].lower(), i))
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
148 if ws[pos:pos+li].lower() == i and not ws[pos+li].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
149 return True
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
150
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
151 def _goto_lt(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
152 self._ltpos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
153 self._ltstate = self._state
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
154 self._state = self._seen_lt
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
155
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
156 def _norm(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
157 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
158 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
159 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
160 if char == "<":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
161 self._goto_lt()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
162 elif char == "\"":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
163 # opening double quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
164 ws[pos] = LDQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
165 self._state = self._seen_ld
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
166 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
167 # in this state, ' is always an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
168 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
169
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
170 def _gettag(self, start):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
171 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
172 end = start
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
173 while ws[end].isalnum():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
174 end += 1
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
175 return ws[start:end].lower()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
176
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
177 def _seen_lt(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
178 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
179 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
180 if ws[pos] == ">":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
181 start = self._ltpos + 1
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
182 if ws[start] == '/':
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
183 if self._gettag(start + 1) in _BLOCK:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
184 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
185 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
186 self._state = self._ltstate
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
187 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
188 tag = self._gettag(start)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
189 if tag in _BLOCK:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
190 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
191 elif tag in _RAW:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
192 self._state = self._raw
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
193 self._endtag = "</" + tag
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
194 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
195 self._state = self._ltstate
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
196
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
197 def _raw(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
198 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
199 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
200 end = pos + len(self._endtag)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
201 # only a matching end tag gets us out of the raw state
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
202 if ws[pos] == '<' and ws[pos:end].lower() == self._endtag and (not ws[end].isalnum()):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
203 self._ltpos = pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
204 self._state = self._seen_lt
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
205
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
206 def _seen_ld(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
207 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
208 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
209 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
210 if char == "<":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
211 self._goto_lt()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
212 elif char == "\"":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
213 # closing double quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
214 ws[pos] = RDQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
215 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
216 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
217 if ws[pos-1].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
218 # either an inter-word, or an end of word, apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
219 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
220 elif ws[pos+1].isdecimal() or _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
221 # also an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
222 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
223 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
224 # opening single quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
225 ws[pos] = LSQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
226 self._state = self._seen_ls
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
227
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
228 def _seen_ls():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
229 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
230 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
231 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
232 if char == "<":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
233 self._goto_lt()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
234 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
235 if ws[pos-1].isalpha() and ws[pos+1].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
236 # obvious apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
237 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
238 elif ws[pos+1].isdecimal() or _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
239 # also an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
240 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
241 elif ws[pos-1].isspace():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
242 # start of word apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
243 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
244 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
245 # closing single quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
246 ws[pos] = RSQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
247 self._state = self._seen_ld