Robots.txt and sitemap.txt

build.py

@@ -19,6 +19,14 @@ PAGES = {

                                
                                
                                
                            
                                
                                    
                                        
                                            19  
                                                "404.html": "404.html"
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            20  
                                            }
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            21  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        22 +
                                        DISALLOWED_SITEMAP = [
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        23 +
                                            "404.html",
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        24 +
                                            "blog-feed.rss"
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        25 +
                                        ]
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        26 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        27 +
                                        SITEMAP_HREF = "https://steve0greatness.github.io/"
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        28 +
                                        sitemap = []
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        29 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            30  
                                            def WipeFinalDir():
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            31  
                                                if not PathExists(BUILD_DIRECTORY):
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            32  
                                                    print("Directory didn't existing, creating it...")
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        

@@ -71,9 +79,12 @@ def RenderPosts():

                                
                                
                                
                            
                                
                                    
                                        
                                            79  
                                                    print("Copying blog/%s to %s/blog/%s" % (post, BUILD_DIRECTORY, PlaintextPath))
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            80  
                                                    with open(BUILD_DIRECTORY + "/blog/" + PlaintextPath, "w", encoding="utf-8") as PostPlaintext:
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            81  
                                                        PostPlaintext.write(PostMD)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        82 +
                                                sitemap.append(SITEMAP_HREF + "/blog/" + PostPath)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            83  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        75 -
                                        def RenderPage(PageInput: str, ContentDest: str, **kwargs):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        84 +
                                        def RenderPage(PageInput: str, ContentDest: str, AllowSitemap: bool = True, **kwargs):
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            85  
                                                print("Building views/%s to %s/%s" % (PageInput, BUILD_DIRECTORY, ContentDest))
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        86 +
                                            if AllowSitemap:
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        87 +
                                                sitemap.append(SITEMAP_HREF + ContentDest)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            88  
                                                with open(BUILD_DIRECTORY + "/" + ContentDest, "w", encoding="utf-8") as DestLocation:
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            89  
                                                    DestLocation.write(RenderTemplate(PageInput, **kwargs))
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            90  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        

@@ -89,6 +100,12 @@ if __name__ == "__main__":

                                
                                
                                
                            
                                
                                    
                                        
                                            100  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            101  
                                                print("Building pages")
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            102  
                                                for file, path in PAGES.items():
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        103 +
                                                if file in DISALLOWED_SITEMAP:
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        104 +
                                                    RenderPage(file, path, False, PostList=PostList)
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        105 +
                                                    continue
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            106  
                                                    RenderPage(file, path, PostList=PostList)
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            107  
                                            
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        108 +
                                            with open(BUILD_DIRECTORY + "/sitemap.txt", "w") as SitemapFile:
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        109 +
                                                SitemapFile.write("\n".join(sitemap))
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        110 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                            111  
                                                pass
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                
                                
                                
                            
                                
                                    
                                        
                                        
                                            
                                            
                                            
                                            
                                        
                                    
                                
                                
                                
                            
                            
                                

static/robots.txt

@@ -0,0 +1,10 @@

                                
                                
                                
                            
                                
                                    
                                        1 +
                                        User-agent: *
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        2 +
                                        Disallow: /src/
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        3 +
                                        Disallow: /favicon.ico
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        4 +
                                        Disallow: /.nojekyll
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        5 +
                                        Disallow: /md-src.svg
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        6 +
                                        Disallow: /rss-feed.png
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        7 +
                                        Disallow: /toot-kytta-dev-icon.svg
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        8 +
                                        Disallow: /blog/feed.rss
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        9 +
                                        
                                        
                                        
                                        
                                    
                                
                                
                                
                            
                                
                                    
                                        10 +
                                        sitemap: https://steve0greatness.github.io