More robust markdown parser

markdown.py

@@ -61,11 +61,9 @@ class Heading(Container):

                                
                                
                                
                            
                                
                                    
                                        
                                            61  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            62  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            63  
                                            class Paragraph(Container):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        64 -
                                            def __init__(self):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        64 +
                                            def __init__(self, content):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            65  
                                                    super().__init__("")
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        66 -
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        67 -
                                            def addLine(self, content):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        68 -
                                                self.content.extend([*parse_line(content), " "])
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        66 +
                                                self.content = parse_line(content)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            67  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            68  
                                                def __repr__(self):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            69  
                                                    return "Paragraph:\n\t" + repr(self.content)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        

@@ -75,6 +73,19 @@ class Paragraph(Container):

                                
                                
                                
                            
                                
                                    
                                        
                                            73  
                                                    return "p"
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            74  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            75  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        76 +
                                        class Blockquote(Paragraph):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        77 +
                                            def __init__(self, content):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        78 +
                                                super().__init__("")
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        79 +
                                                self.content = tokenise(content)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        80 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        81 +
                                            def __repr__(self):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        82 +
                                                return "Blockquote:\n\t" + repr(self.content)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        83 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        84 +
                                            @property
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        85 +
                                            def tag_name(self):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        86 +
                                                return "blockquote"
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        87 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        88 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            89  
                                            class Emphasis(Container):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            90  
                                                def __init__(self, content, value):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            91  
                                                    super().__init__(content)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        

@@ -140,7 +151,7 @@ class Link(Element):

                                
                                
                                
                            
                                
                                    
                                        
                                            151  
                                                    self.image = image
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            152  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            153  
                                                def __repr__(self):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        143 -
                                                return f"{'Image' if self.image else 'Link'}: {self.text} -> {self.destination}"
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        154 +
                                                return f"{'Image' if self.image else 'Link'}: {self.content} -> {self.destination}"
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            155  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            156  
                                                @property
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            157  
                                                def tag_name(self):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        

@@ -182,9 +193,10 @@ def parse_line(source):

                                
                                
                                
                            
                                
                                    
                                        
                                            193  
                                                    if i.group("diff"):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            194  
                                                        tokens.append(Diff(i.group("textDiff"), i.group("diff")))
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            195  
                                                    if i.group("urlText"):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        185 -
                                                    tokens.append(Link(i.group("urlText"), i.group("urlDestination")))
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        186 -
                                                if i.group("imageFlag"):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        187 -
                                                    tokens.append(Image(i.group("urlText"), i.group("urlDestination")))
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        196 +
                                                    if i.group("imageFlag"):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        197 +
                                                        tokens.append(Image(i.group("urlText"), i.group("urlDestination")))
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        198 +
                                                    else:
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        199 +
                                                        tokens.append(Link(i.group("urlText"), i.group("urlDestination")))
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            200  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            201  
                                                tokens.append(source[lookup:])
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            202  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        

@@ -194,27 +206,51 @@ def parse_line(source):

                                
                                
                                
                            
                                
                                    
                                        
                                            206  
                                            def tokenise(source):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            207  
                                                tokens = []
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            208  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        197 -
                                            current_block = Element
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        209 +
                                            current_block = Element()
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        210 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        211 +
                                            lines = source.split("\n")
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            212  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        199 -
                                            for line in source.split("\n"):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        213 +
                                            i = 0
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        214 +
                                            while i < len(lines):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        215 +
                                                line = lines[i]
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            216  
                                                    if not line.strip():
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            217  
                                                        # Void block
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            218  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            219  
                                                        tokens.append(current_block)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            220  
                                                        current_block = Element()
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        221 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        222 +
                                                    i += 1
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            223  
                                                    elif line.startswith("#") and leading(line.lstrip("#"), " "):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            224  
                                                        tokens.append(current_block)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            225  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            226  
                                                        content = line.lstrip("#").strip()
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            227  
                                                        current_block = Heading(content, leading(line, "#"))
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        228 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        229 +
                                                    i += 1
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        230 +
                                                elif line.startswith(">"):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        231 +
                                                    if not isinstance(current_block, Blockquote):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        232 +
                                                        tokens.append(current_block)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        233 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        234 +
                                                    content = ""
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        235 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        236 +
                                                    while i < len(lines) and lines[i].startswith(">"):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        237 +
                                                        content += lines[i].lstrip(">").strip() + "\n"
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        238 +
                                                        i += 1
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        239 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        240 +
                                                    current_block = Blockquote(content)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            241  
                                                    else:
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            242  
                                                        if not isinstance(current_block, Paragraph):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            243  
                                                            # Paragraph is default
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            244  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            245  
                                                            tokens.append(current_block)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        215 -
                                                        current_block = Paragraph()
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            246  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        217 -
                                                    current_block.addLine(line.strip())
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        247 +
                                                    content = ""
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        248 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        249 +
                                                    while i < len(lines) and not lines[i].startswith("#") and not lines[i].startswith(">") and lines[i].strip():
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        250 +
                                                        content += lines[i].strip() + "\n"
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        251 +
                                                        i += 1
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        252 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        253 +
                                                    current_block = Paragraph(content)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            254  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            255  
                                                tokens.append(current_block)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            256  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        

@@ -229,6 +265,10 @@ def make_html(ast):

                                
                                
                                
                            
                                
                                    
                                        
                                            265  
                                                        soup.append(i)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            266  
                                                    elif hasattr(i, "content") and i.tag_name != "m-void":
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            267  
                                                        tag = soup.new_tag(str(i.tag_name))
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        268 +
                                                    if i.tag_name == "a":
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        269 +
                                                        tag["href"] = i.destination
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        270 +
                                                    if i.tag_name == "img":
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        271 +
                                                        tag["src"] = i.destination
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            272  
                                                        try:
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            273  
                                                            if isinstance(i.content, list):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            274  
                                                                tag.append(make_html(i.content))
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        

@@ -255,11 +295,14 @@ if __name__ == '__main__':

                                
                                
                                
                            
                                
                                    
                                        
                                            295  
                                            Lorem **i`p`sum**
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            296  
                                            dolor `sit` amet
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            297  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        298 +
                                        > Make it as simple as possible, [but not simpler](https://wikipedia.org).
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        299 +
                                        > > If you can't explain it simply, you don't understand it well enough.
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        300 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            301  
                                            ...
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            302  
                                            """
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            303  
                                                )
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        261 -
                                            # for i in ast:
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        262 -
                                            #     print(repr(i))
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        304 +
                                            for i in ast:
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        305 +
                                                print(repr(i))
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            306  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            307  
                                                # Now convert the AST to HTML
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            308  
                                                print(make_html(ast).prettify(formatter=beautifulsoup.formatter.HTMLFormatter(indent=4)))