annotate curlers.py @ 4:7a83e82e65a6

Remove some deadwood.
author David Barts <n5jrn@me.com>
date Thu, 26 Dec 2019 20:04:04 -0800
parents 091c03f1b2e8
children da3fb2312c88
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
3
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
1 #!/usr/bin/env python3
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
2 # -*- coding: utf-8 -*-
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
3
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
4 # Classes for curling both HTML and plain text.
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
5
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
6 # I m p o r t s
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
7
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
8 import os, sys
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
9 from workspace import Workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
10
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
11 # V a r i a b l e s
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
12
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
13 # Quote types
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
14 LSQUO = "\u2018"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
15 APOS = RSQUO = "\u2019"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
16 LDQUO = "\u201C"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
17 RDQUO = "\u201D"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
18
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
19 # Words that start with an apostrophe. Cribbed from Wordpress.
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
20 _ASTART = [ "'tain't", "'twere", "'twas", "'tis", "'twill", "'til",
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
21 "'bout", "'nuff", "'round", "'cause" , "'em" ]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
22
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
23 # HTML tags that enclose raw data
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
24 _RAW = set(["script", "style"])
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
25
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
26 # HTML block elements
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
27 _BLOCK = set([
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
28 "address", "blockquote", "div", "dl", "fieldset", "form", "h1",
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
29 "h2", "h3", "h4", "h5", "h6", "hr", "noscript", "ol", "p", "pre",
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
30 "table", "ul"
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
31 ])
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
32
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
33 # F u n c t i o n s
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
34
4
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
35 def uncurl(ws):
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
36 """
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
37 Makes all quotes in the workspace non-curly.
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
38 """
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
39 for i in range(len(ws)):
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
40 ch = ws[i]
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
41 if ch in set([LDQUO, RDQUO]):
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
42 ws[i] = '"'
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
43 elif ch in set([LSQUO, RSQUO]):
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
44 ws[i] = "'"
7a83e82e65a6 Remove some deadwood.
David Barts <n5jrn@me.com>
parents: 3
diff changeset
45
3
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
46 def _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
47 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
48 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
49 for i in _ASTART:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
50 li = len(i)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
51 if ws[pos:pos+li].lower() == i and not ws[pos+li].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
52 return True
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
53
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
54 # C l a s s e s
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
55
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
56 class BaseCurler():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
57 def feed(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
58 raise NotImplementedError()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
59
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
60 class TextCurler(BaseCurler):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
61 """
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
62 For processing plain text. Assumes the entire text is a block; it is
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
63 the responsibility of the caller to break the input into paragraphs.
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
64 """
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
65 def __init__(self, workspace):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
66 self.workspace = workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
67 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
68 self._pos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
69
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
70 def feed(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
71 self._pos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
72 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
73 for self._pos in range(len(self.workspace)):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
74 self._state()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
75
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
76 def _is_cockney(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
77 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
78 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
79 for i in _ASTART:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
80 li = len(i)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
81 print("comparing {0!r} and {1!r}\n".format(ws[pos:pos+li].lower(), i))
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
82 if ws[pos:pos+li].lower() == i and not ws[pos+li].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
83 return True
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
84
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
85 def _norm(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
86 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
87 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
88 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
89 if char == "\"":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
90 # opening double quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
91 ws[pos] = LDQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
92 self._state = self._seen_ld
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
93 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
94 # in this state, ' is always an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
95 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
96
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
97 def _seen_ld(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
98 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
99 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
100 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
101 if char == "\"":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
102 # closing double quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
103 ws[pos] = RDQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
104 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
105 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
106 if ws[pos-1].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
107 # either an inter-word, or an end of word, apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
108 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
109 elif ws[pos+1].isdecimal() or _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
110 # also an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
111 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
112 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
113 # opening single quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
114 ws[pos] = LSQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
115 self._state = self._seen_ls
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
116
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
117 def _seen_ls(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
118 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
119 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
120 if ws[pos] == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
121 if ws[pos-1].isalpha() and ws[pos+1].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
122 # obvious apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
123 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
124 elif ws[pos+1].isdecimal() or _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
125 # also an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
126 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
127 elif ws[pos-1].isspace():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
128 # start of word apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
129 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
130 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
131 # closing single quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
132 ws[pos] = RSQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
133 self._state = self._seen_ld
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
134
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
135 class HtmlCurler(BaseCurler):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
136 """
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
137 For processing HTML. Uses HTML block tags to delimit blocks.
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
138 """
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
139 def __init__(self, workspace):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
140 self.workspace = workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
141 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
142 self._pos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
143 self._ltpos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
144 self._endtag = None
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
145 self._ltstate = None
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
146
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
147 def feed(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
148 self._pos = 0
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
149 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
150 for self._pos in range(len(self.workspace)):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
151 self._state()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
152
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
153 def _is_cockney(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
154 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
155 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
156 for i in _ASTART:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
157 li = len(i)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
158 print("comparing {0!r} and {1!r}\n".format(ws[pos:pos+li].lower(), i))
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
159 if ws[pos:pos+li].lower() == i and not ws[pos+li].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
160 return True
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
161
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
162 def _goto_lt(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
163 self._ltpos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
164 self._ltstate = self._state
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
165 self._state = self._seen_lt
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
166
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
167 def _norm(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
168 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
169 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
170 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
171 if char == "<":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
172 self._goto_lt()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
173 elif char == "\"":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
174 # opening double quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
175 ws[pos] = LDQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
176 self._state = self._seen_ld
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
177 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
178 # in this state, ' is always an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
179 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
180
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
181 def _gettag(self, start):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
182 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
183 end = start
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
184 while ws[end].isalnum():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
185 end += 1
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
186 return ws[start:end].lower()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
187
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
188 def _seen_lt(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
189 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
190 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
191 if ws[pos] == ">":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
192 start = self._ltpos + 1
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
193 if ws[start] == '/':
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
194 if self._gettag(start + 1) in _BLOCK:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
195 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
196 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
197 self._state = self._ltstate
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
198 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
199 tag = self._gettag(start)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
200 if tag in _BLOCK:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
201 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
202 elif tag in _RAW:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
203 self._state = self._raw
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
204 self._endtag = "</" + tag
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
205 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
206 self._state = self._ltstate
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
207
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
208 def _raw(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
209 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
210 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
211 end = pos + len(self._endtag)
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
212 # only a matching end tag gets us out of the raw state
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
213 if ws[pos] == '<' and ws[pos:end].lower() == self._endtag and (not ws[end].isalnum()):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
214 self._ltpos = pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
215 self._state = self._seen_lt
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
216
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
217 def _seen_ld(self):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
218 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
219 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
220 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
221 if char == "<":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
222 self._goto_lt()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
223 elif char == "\"":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
224 # closing double quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
225 ws[pos] = RDQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
226 self._state = self._norm
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
227 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
228 if ws[pos-1].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
229 # either an inter-word, or an end of word, apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
230 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
231 elif ws[pos+1].isdecimal() or _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
232 # also an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
233 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
234 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
235 # opening single quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
236 ws[pos] = LSQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
237 self._state = self._seen_ls
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
238
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
239 def _seen_ls():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
240 pos = self._pos
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
241 ws = self.workspace
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
242 char = ws[pos]
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
243 if char == "<":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
244 self._goto_lt()
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
245 elif char == "'":
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
246 if ws[pos-1].isalpha() and ws[pos+1].isalpha():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
247 # obvious apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
248 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
249 elif ws[pos+1].isdecimal() or _is_cockney(pos, ws):
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
250 # also an apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
251 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
252 elif ws[pos-1].isspace():
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
253 # start of word apostrophe
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
254 ws[pos] = APOS
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
255 else:
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
256 # closing single quote
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
257 ws[pos] = RSQUO
091c03f1b2e8 Getting it working...
David Barts <n5jrn@me.com>
parents:
diff changeset
258 self._state = self._seen_ld