OLD | NEW |
| (Empty) |
1 """ | |
2 CORE MARKDOWN BLOCKPARSER | |
3 =========================================================================== | |
4 | |
5 This parser handles basic parsing of Markdown blocks. It doesn't concern | |
6 itself with inline elements such as **bold** or *italics*, but rather just | |
7 catches blocks, lists, quotes, etc. | |
8 | |
9 The BlockParser is made up of a bunch of BlockProssors, each handling a | |
10 different type of block. Extensions may add/replace/remove BlockProcessors | |
11 as they need to alter how markdown blocks are parsed. | |
12 """ | |
13 | |
14 from __future__ import absolute_import | |
15 from __future__ import division | |
16 from __future__ import unicode_literals | |
17 import logging | |
18 import re | |
19 from . import util | |
20 from .blockparser import BlockParser | |
21 | |
22 logger = logging.getLogger('MARKDOWN') | |
23 | |
24 | |
25 def build_block_parser(md_instance, **kwargs): | |
26 """ Build the default block parser used by Markdown. """ | |
27 parser = BlockParser(md_instance) | |
28 parser.blockprocessors['empty'] = EmptyBlockProcessor(parser) | |
29 parser.blockprocessors['indent'] = ListIndentProcessor(parser) | |
30 parser.blockprocessors['code'] = CodeBlockProcessor(parser) | |
31 parser.blockprocessors['hashheader'] = HashHeaderProcessor(parser) | |
32 parser.blockprocessors['setextheader'] = SetextHeaderProcessor(parser) | |
33 parser.blockprocessors['hr'] = HRProcessor(parser) | |
34 parser.blockprocessors['olist'] = OListProcessor(parser) | |
35 parser.blockprocessors['ulist'] = UListProcessor(parser) | |
36 parser.blockprocessors['quote'] = BlockQuoteProcessor(parser) | |
37 parser.blockprocessors['paragraph'] = ParagraphProcessor(parser) | |
38 return parser | |
39 | |
40 | |
41 class BlockProcessor: | |
42 """ Base class for block processors. | |
43 | |
44 Each subclass will provide the methods below to work with the source and | |
45 tree. Each processor will need to define it's own ``test`` and ``run`` | |
46 methods. The ``test`` method should return True or False, to indicate | |
47 whether the current block should be processed by this processor. If the | |
48 test passes, the parser will call the processors ``run`` method. | |
49 | |
50 """ | |
51 | |
52 def __init__(self, parser): | |
53 self.parser = parser | |
54 self.tab_length = parser.markdown.tab_length | |
55 | |
56 def lastChild(self, parent): | |
57 """ Return the last child of an etree element. """ | |
58 if len(parent): | |
59 return parent[-1] | |
60 else: | |
61 return None | |
62 | |
63 def detab(self, text): | |
64 """ Remove a tab from the front of each line of the given text. """ | |
65 newtext = [] | |
66 lines = text.split('\n') | |
67 for line in lines: | |
68 if line.startswith(' '*self.tab_length): | |
69 newtext.append(line[self.tab_length:]) | |
70 elif not line.strip(): | |
71 newtext.append('') | |
72 else: | |
73 break | |
74 return '\n'.join(newtext), '\n'.join(lines[len(newtext):]) | |
75 | |
76 def looseDetab(self, text, level=1): | |
77 """ Remove a tab from front of lines but allowing dedented lines. """ | |
78 lines = text.split('\n') | |
79 for i in range(len(lines)): | |
80 if lines[i].startswith(' '*self.tab_length*level): | |
81 lines[i] = lines[i][self.tab_length*level:] | |
82 return '\n'.join(lines) | |
83 | |
84 def test(self, parent, block): | |
85 """ Test for block type. Must be overridden by subclasses. | |
86 | |
87 As the parser loops through processors, it will call the ``test`` | |
88 method on each to determine if the given block of text is of that | |
89 type. This method must return a boolean ``True`` or ``False``. The | |
90 actual method of testing is left to the needs of that particular | |
91 block type. It could be as simple as ``block.startswith(some_string)`` | |
92 or a complex regular expression. As the block type may be different | |
93 depending on the parent of the block (i.e. inside a list), the parent | |
94 etree element is also provided and may be used as part of the test. | |
95 | |
96 Keywords: | |
97 | |
98 * ``parent``: A etree element which will be the parent of the block. | |
99 * ``block``: A block of text from the source which has been split at | |
100 blank lines. | |
101 """ | |
102 pass # pragma: no cover | |
103 | |
104 def run(self, parent, blocks): | |
105 """ Run processor. Must be overridden by subclasses. | |
106 | |
107 When the parser determines the appropriate type of a block, the parser | |
108 will call the corresponding processor's ``run`` method. This method | |
109 should parse the individual lines of the block and append them to | |
110 the etree. | |
111 | |
112 Note that both the ``parent`` and ``etree`` keywords are pointers | |
113 to instances of the objects which should be edited in place. Each | |
114 processor must make changes to the existing objects as there is no | |
115 mechanism to return new/different objects to replace them. | |
116 | |
117 This means that this method should be adding SubElements or adding text | |
118 to the parent, and should remove (``pop``) or add (``insert``) items to | |
119 the list of blocks. | |
120 | |
121 Keywords: | |
122 | |
123 * ``parent``: A etree element which is the parent of the current block. | |
124 * ``blocks``: A list of all remaining blocks of the document. | |
125 """ | |
126 pass # pragma: no cover | |
127 | |
128 | |
129 class ListIndentProcessor(BlockProcessor): | |
130 """ Process children of list items. | |
131 | |
132 Example: | |
133 * a list item | |
134 process this part | |
135 | |
136 or this part | |
137 | |
138 """ | |
139 | |
140 ITEM_TYPES = ['li'] | |
141 LIST_TYPES = ['ul', 'ol'] | |
142 | |
143 def __init__(self, *args): | |
144 BlockProcessor.__init__(self, *args) | |
145 self.INDENT_RE = re.compile(r'^(([ ]{%s})+)' % self.tab_length) | |
146 | |
147 def test(self, parent, block): | |
148 return block.startswith(' '*self.tab_length) and \ | |
149 not self.parser.state.isstate('detabbed') and \ | |
150 (parent.tag in self.ITEM_TYPES or | |
151 (len(parent) and parent[-1] is not None and | |
152 (parent[-1].tag in self.LIST_TYPES))) | |
153 | |
154 def run(self, parent, blocks): | |
155 block = blocks.pop(0) | |
156 level, sibling = self.get_level(parent, block) | |
157 block = self.looseDetab(block, level) | |
158 | |
159 self.parser.state.set('detabbed') | |
160 if parent.tag in self.ITEM_TYPES: | |
161 # It's possible that this parent has a 'ul' or 'ol' child list | |
162 # with a member. If that is the case, then that should be the | |
163 # parent. This is intended to catch the edge case of an indented | |
164 # list whose first member was parsed previous to this point | |
165 # see OListProcessor | |
166 if len(parent) and parent[-1].tag in self.LIST_TYPES: | |
167 self.parser.parseBlocks(parent[-1], [block]) | |
168 else: | |
169 # The parent is already a li. Just parse the child block. | |
170 self.parser.parseBlocks(parent, [block]) | |
171 elif sibling.tag in self.ITEM_TYPES: | |
172 # The sibling is a li. Use it as parent. | |
173 self.parser.parseBlocks(sibling, [block]) | |
174 elif len(sibling) and sibling[-1].tag in self.ITEM_TYPES: | |
175 # The parent is a list (``ol`` or ``ul``) which has children. | |
176 # Assume the last child li is the parent of this block. | |
177 if sibling[-1].text: | |
178 # If the parent li has text, that text needs to be moved to a p | |
179 # The p must be 'inserted' at beginning of list in the event | |
180 # that other children already exist i.e.; a nested sublist. | |
181 p = util.etree.Element('p') | |
182 p.text = sibling[-1].text | |
183 sibling[-1].text = '' | |
184 sibling[-1].insert(0, p) | |
185 self.parser.parseChunk(sibling[-1], block) | |
186 else: | |
187 self.create_item(sibling, block) | |
188 self.parser.state.reset() | |
189 | |
190 def create_item(self, parent, block): | |
191 """ Create a new li and parse the block with it as the parent. """ | |
192 li = util.etree.SubElement(parent, 'li') | |
193 self.parser.parseBlocks(li, [block]) | |
194 | |
195 def get_level(self, parent, block): | |
196 """ Get level of indent based on list level. """ | |
197 # Get indent level | |
198 m = self.INDENT_RE.match(block) | |
199 if m: | |
200 indent_level = len(m.group(1))/self.tab_length | |
201 else: | |
202 indent_level = 0 | |
203 if self.parser.state.isstate('list'): | |
204 # We're in a tightlist - so we already are at correct parent. | |
205 level = 1 | |
206 else: | |
207 # We're in a looselist - so we need to find parent. | |
208 level = 0 | |
209 # Step through children of tree to find matching indent level. | |
210 while indent_level > level: | |
211 child = self.lastChild(parent) | |
212 if (child is not None and | |
213 (child.tag in self.LIST_TYPES or child.tag in self.ITEM_TYPES)): | |
214 if child.tag in self.LIST_TYPES: | |
215 level += 1 | |
216 parent = child | |
217 else: | |
218 # No more child levels. If we're short of indent_level, | |
219 # we have a code block. So we stop here. | |
220 break | |
221 return level, parent | |
222 | |
223 | |
224 class CodeBlockProcessor(BlockProcessor): | |
225 """ Process code blocks. """ | |
226 | |
227 def test(self, parent, block): | |
228 return block.startswith(' '*self.tab_length) | |
229 | |
230 def run(self, parent, blocks): | |
231 sibling = self.lastChild(parent) | |
232 block = blocks.pop(0) | |
233 theRest = '' | |
234 if (sibling is not None and sibling.tag == "pre" and | |
235 len(sibling) and sibling[0].tag == "code"): | |
236 # The previous block was a code block. As blank lines do not start | |
237 # new code blocks, append this block to the previous, adding back | |
238 # linebreaks removed from the split into a list. | |
239 code = sibling[0] | |
240 block, theRest = self.detab(block) | |
241 code.text = util.AtomicString( | |
242 '%s\n%s\n' % (code.text, block.rstrip()) | |
243 ) | |
244 else: | |
245 # This is a new codeblock. Create the elements and insert text. | |
246 pre = util.etree.SubElement(parent, 'pre') | |
247 code = util.etree.SubElement(pre, 'code') | |
248 block, theRest = self.detab(block) | |
249 code.text = util.AtomicString('%s\n' % block.rstrip()) | |
250 if theRest: | |
251 # This block contained unindented line(s) after the first indented | |
252 # line. Insert these lines as the first block of the master blocks | |
253 # list for future processing. | |
254 blocks.insert(0, theRest) | |
255 | |
256 | |
257 class BlockQuoteProcessor(BlockProcessor): | |
258 | |
259 RE = re.compile(r'(^|\n)[ ]{0,3}>[ ]?(.*)') | |
260 | |
261 def test(self, parent, block): | |
262 return bool(self.RE.search(block)) | |
263 | |
264 def run(self, parent, blocks): | |
265 block = blocks.pop(0) | |
266 m = self.RE.search(block) | |
267 if m: | |
268 before = block[:m.start()] # Lines before blockquote | |
269 # Pass lines before blockquote in recursively for parsing forst. | |
270 self.parser.parseBlocks(parent, [before]) | |
271 # Remove ``> `` from begining of each line. | |
272 block = '\n'.join( | |
273 [self.clean(line) for line in block[m.start():].split('\n')] | |
274 ) | |
275 sibling = self.lastChild(parent) | |
276 if sibling is not None and sibling.tag == "blockquote": | |
277 # Previous block was a blockquote so set that as this blocks parent | |
278 quote = sibling | |
279 else: | |
280 # This is a new blockquote. Create a new parent element. | |
281 quote = util.etree.SubElement(parent, 'blockquote') | |
282 # Recursively parse block with blockquote as parent. | |
283 # change parser state so blockquotes embedded in lists use p tags | |
284 self.parser.state.set('blockquote') | |
285 self.parser.parseChunk(quote, block) | |
286 self.parser.state.reset() | |
287 | |
288 def clean(self, line): | |
289 """ Remove ``>`` from beginning of a line. """ | |
290 m = self.RE.match(line) | |
291 if line.strip() == ">": | |
292 return "" | |
293 elif m: | |
294 return m.group(2) | |
295 else: | |
296 return line | |
297 | |
298 | |
299 class OListProcessor(BlockProcessor): | |
300 """ Process ordered list blocks. """ | |
301 | |
302 TAG = 'ol' | |
303 # Detect an item (``1. item``). ``group(1)`` contains contents of item. | |
304 RE = re.compile(r'^[ ]{0,3}\d+\.[ ]+(.*)') | |
305 # Detect items on secondary lines. they can be of either list type. | |
306 CHILD_RE = re.compile(r'^[ ]{0,3}((\d+\.)|[*+-])[ ]+(.*)') | |
307 # Detect indented (nested) items of either type | |
308 INDENT_RE = re.compile(r'^[ ]{4,7}((\d+\.)|[*+-])[ ]+.*') | |
309 # The integer (python string) with which the lists starts (default=1) | |
310 # Eg: If list is intialized as) | |
311 # 3. Item | |
312 # The ol tag will get starts="3" attribute | |
313 STARTSWITH = '1' | |
314 # List of allowed sibling tags. | |
315 SIBLING_TAGS = ['ol', 'ul'] | |
316 | |
317 def test(self, parent, block): | |
318 return bool(self.RE.match(block)) | |
319 | |
320 def run(self, parent, blocks): | |
321 # Check fr multiple items in one block. | |
322 items = self.get_items(blocks.pop(0)) | |
323 sibling = self.lastChild(parent) | |
324 | |
325 if sibling is not None and sibling.tag in self.SIBLING_TAGS: | |
326 # Previous block was a list item, so set that as parent | |
327 lst = sibling | |
328 # make sure previous item is in a p- if the item has text, | |
329 # then it isn't in a p | |
330 if lst[-1].text: | |
331 # since it's possible there are other children for this | |
332 # sibling, we can't just SubElement the p, we need to | |
333 # insert it as the first item. | |
334 p = util.etree.Element('p') | |
335 p.text = lst[-1].text | |
336 lst[-1].text = '' | |
337 lst[-1].insert(0, p) | |
338 # if the last item has a tail, then the tail needs to be put in a p | |
339 # likely only when a header is not followed by a blank line | |
340 lch = self.lastChild(lst[-1]) | |
341 if lch is not None and lch.tail: | |
342 p = util.etree.SubElement(lst[-1], 'p') | |
343 p.text = lch.tail.lstrip() | |
344 lch.tail = '' | |
345 | |
346 # parse first block differently as it gets wrapped in a p. | |
347 li = util.etree.SubElement(lst, 'li') | |
348 self.parser.state.set('looselist') | |
349 firstitem = items.pop(0) | |
350 self.parser.parseBlocks(li, [firstitem]) | |
351 self.parser.state.reset() | |
352 elif parent.tag in ['ol', 'ul']: | |
353 # this catches the edge case of a multi-item indented list whose | |
354 # first item is in a blank parent-list item: | |
355 # * * subitem1 | |
356 # * subitem2 | |
357 # see also ListIndentProcessor | |
358 lst = parent | |
359 else: | |
360 # This is a new list so create parent with appropriate tag. | |
361 lst = util.etree.SubElement(parent, self.TAG) | |
362 # Check if a custom start integer is set | |
363 if not self.parser.markdown.lazy_ol and self.STARTSWITH != '1': | |
364 lst.attrib['start'] = self.STARTSWITH | |
365 | |
366 self.parser.state.set('list') | |
367 # Loop through items in block, recursively parsing each with the | |
368 # appropriate parent. | |
369 for item in items: | |
370 if item.startswith(' '*self.tab_length): | |
371 # Item is indented. Parse with last item as parent | |
372 self.parser.parseBlocks(lst[-1], [item]) | |
373 else: | |
374 # New item. Create li and parse with it as parent | |
375 li = util.etree.SubElement(lst, 'li') | |
376 self.parser.parseBlocks(li, [item]) | |
377 self.parser.state.reset() | |
378 | |
379 def get_items(self, block): | |
380 """ Break a block into list items. """ | |
381 items = [] | |
382 for line in block.split('\n'): | |
383 m = self.CHILD_RE.match(line) | |
384 if m: | |
385 # This is a new list item | |
386 # Check first item for the start index | |
387 if not items and self.TAG == 'ol': | |
388 # Detect the integer value of first list item | |
389 INTEGER_RE = re.compile('(\d+)') | |
390 self.STARTSWITH = INTEGER_RE.match(m.group(1)).group() | |
391 # Append to the list | |
392 items.append(m.group(3)) | |
393 elif self.INDENT_RE.match(line): | |
394 # This is an indented (possibly nested) item. | |
395 if items[-1].startswith(' '*self.tab_length): | |
396 # Previous item was indented. Append to that item. | |
397 items[-1] = '%s\n%s' % (items[-1], line) | |
398 else: | |
399 items.append(line) | |
400 else: | |
401 # This is another line of previous item. Append to that item. | |
402 items[-1] = '%s\n%s' % (items[-1], line) | |
403 return items | |
404 | |
405 | |
406 class UListProcessor(OListProcessor): | |
407 """ Process unordered list blocks. """ | |
408 | |
409 TAG = 'ul' | |
410 RE = re.compile(r'^[ ]{0,3}[*+-][ ]+(.*)') | |
411 | |
412 | |
413 class HashHeaderProcessor(BlockProcessor): | |
414 """ Process Hash Headers. """ | |
415 | |
416 # Detect a header at start of any line in block | |
417 RE = re.compile(r'(^|\n)(?P<level>#{1,6})(?P<header>.*?)#*(\n|$)') | |
418 | |
419 def test(self, parent, block): | |
420 return bool(self.RE.search(block)) | |
421 | |
422 def run(self, parent, blocks): | |
423 block = blocks.pop(0) | |
424 m = self.RE.search(block) | |
425 if m: | |
426 before = block[:m.start()] # All lines before header | |
427 after = block[m.end():] # All lines after header | |
428 if before: | |
429 # As the header was not the first line of the block and the | |
430 # lines before the header must be parsed first, | |
431 # recursively parse this lines as a block. | |
432 self.parser.parseBlocks(parent, [before]) | |
433 # Create header using named groups from RE | |
434 h = util.etree.SubElement(parent, 'h%d' % len(m.group('level'))) | |
435 h.text = m.group('header').strip() | |
436 if after: | |
437 # Insert remaining lines as first block for future parsing. | |
438 blocks.insert(0, after) | |
439 else: # pragma: no cover | |
440 # This should never happen, but just in case... | |
441 logger.warn("We've got a problem header: %r" % block) | |
442 | |
443 | |
444 class SetextHeaderProcessor(BlockProcessor): | |
445 """ Process Setext-style Headers. """ | |
446 | |
447 # Detect Setext-style header. Must be first 2 lines of block. | |
448 RE = re.compile(r'^.*?\n[=-]+[ ]*(\n|$)', re.MULTILINE) | |
449 | |
450 def test(self, parent, block): | |
451 return bool(self.RE.match(block)) | |
452 | |
453 def run(self, parent, blocks): | |
454 lines = blocks.pop(0).split('\n') | |
455 # Determine level. ``=`` is 1 and ``-`` is 2. | |
456 if lines[1].startswith('='): | |
457 level = 1 | |
458 else: | |
459 level = 2 | |
460 h = util.etree.SubElement(parent, 'h%d' % level) | |
461 h.text = lines[0].strip() | |
462 if len(lines) > 2: | |
463 # Block contains additional lines. Add to master blocks for later. | |
464 blocks.insert(0, '\n'.join(lines[2:])) | |
465 | |
466 | |
467 class HRProcessor(BlockProcessor): | |
468 """ Process Horizontal Rules. """ | |
469 | |
470 RE = r'^[ ]{0,3}((-+[ ]{0,2}){3,}|(_+[ ]{0,2}){3,}|(\*+[ ]{0,2}){3,})[ ]*' | |
471 # Detect hr on any line of a block. | |
472 SEARCH_RE = re.compile(RE, re.MULTILINE) | |
473 | |
474 def test(self, parent, block): | |
475 m = self.SEARCH_RE.search(block) | |
476 # No atomic grouping in python so we simulate it here for performance. | |
477 # The regex only matches what would be in the atomic group - the HR. | |
478 # Then check if we are at end of block or if next char is a newline. | |
479 if m and (m.end() == len(block) or block[m.end()] == '\n'): | |
480 # Save match object on class instance so we can use it later. | |
481 self.match = m | |
482 return True | |
483 return False | |
484 | |
485 def run(self, parent, blocks): | |
486 block = blocks.pop(0) | |
487 # Check for lines in block before hr. | |
488 prelines = block[:self.match.start()].rstrip('\n') | |
489 if prelines: | |
490 # Recursively parse lines before hr so they get parsed first. | |
491 self.parser.parseBlocks(parent, [prelines]) | |
492 # create hr | |
493 util.etree.SubElement(parent, 'hr') | |
494 # check for lines in block after hr. | |
495 postlines = block[self.match.end():].lstrip('\n') | |
496 if postlines: | |
497 # Add lines after hr to master blocks for later parsing. | |
498 blocks.insert(0, postlines) | |
499 | |
500 | |
501 class EmptyBlockProcessor(BlockProcessor): | |
502 """ Process blocks that are empty or start with an empty line. """ | |
503 | |
504 def test(self, parent, block): | |
505 return not block or block.startswith('\n') | |
506 | |
507 def run(self, parent, blocks): | |
508 block = blocks.pop(0) | |
509 filler = '\n\n' | |
510 if block: | |
511 # Starts with empty line | |
512 # Only replace a single line. | |
513 filler = '\n' | |
514 # Save the rest for later. | |
515 theRest = block[1:] | |
516 if theRest: | |
517 # Add remaining lines to master blocks for later. | |
518 blocks.insert(0, theRest) | |
519 sibling = self.lastChild(parent) | |
520 if (sibling is not None and sibling.tag == 'pre' and | |
521 len(sibling) and sibling[0].tag == 'code'): | |
522 # Last block is a codeblock. Append to preserve whitespace. | |
523 sibling[0].text = util.AtomicString( | |
524 '%s%s' % (sibling[0].text, filler) | |
525 ) | |
526 | |
527 | |
528 class ParagraphProcessor(BlockProcessor): | |
529 """ Process Paragraph blocks. """ | |
530 | |
531 def test(self, parent, block): | |
532 return True | |
533 | |
534 def run(self, parent, blocks): | |
535 block = blocks.pop(0) | |
536 if block.strip(): | |
537 # Not a blank block. Add to parent, otherwise throw it away. | |
538 if self.parser.state.isstate('list'): | |
539 # The parent is a tight-list. | |
540 # | |
541 # Check for any children. This will likely only happen in a | |
542 # tight-list when a header isn't followed by a blank line. | |
543 # For example: | |
544 # | |
545 # * # Header | |
546 # Line 2 of list item - not part of header. | |
547 sibling = self.lastChild(parent) | |
548 if sibling is not None: | |
549 # Insetrt after sibling. | |
550 if sibling.tail: | |
551 sibling.tail = '%s\n%s' % (sibling.tail, block) | |
552 else: | |
553 sibling.tail = '\n%s' % block | |
554 else: | |
555 # Append to parent.text | |
556 if parent.text: | |
557 parent.text = '%s\n%s' % (parent.text, block) | |
558 else: | |
559 parent.text = block.lstrip() | |
560 else: | |
561 # Create a regular paragraph | |
562 p = util.etree.SubElement(parent, 'p') | |
563 p.text = block.lstrip() | |
OLD | NEW |