added type annotations

blag/blag.py

@@ -10,6 +10,7 @@ import configparser
 import logging
 import os
 import shutil
+import sqlite3
 import sys
 from typing import Any
 
@@ -218,7 +219,6 @@ def build(args: argparse.Namespace) -> None:
     """
     os.makedirs(f"{args.output_dir}", exist_ok=True)
     convertibles = []
-    known_targets = []
     for root, dirnames, filenames in os.walk(args.input_dir):
         for filename in filenames:
             rel_src = os.path.relpath(
@@ -230,22 +230,15 @@ def build(args: argparse.Namespace) -> None:
                 rel_dst = rel_src
                 rel_dst = rel_dst[:-3] + ".html"
                 convertibles.append((rel_src, rel_dst))
-                known_targets.append(
-                    os.path.abspath(f"{args.output_dir}/{rel_dst}")
-                )
             else:
                 shutil.copy(
                     f"{args.input_dir}/{rel_src}",
                     f"{args.output_dir}/{rel_src}",
                 )
-                known_targets.append(
-                    os.path.abspath(f"{args.output_dir}/{rel_src}")
-                )
         for dirname in dirnames:
             # all directories are copied into the output directory
             path = os.path.relpath(f"{root}/{dirname}", start=args.input_dir)
             os.makedirs(f"{args.output_dir}/{path}", exist_ok=True)
-            known_targets.append(os.path.abspath(f"{args.output_dir}/{path}"))
 
     # copy static files over
     logger.info("Copying static files.")
@@ -281,24 +274,6 @@ def build(args: argparse.Namespace) -> None:
         article_template,
     )
 
-    # clean up files that should not be there
-    for root, dirnames, filenames in os.walk(args.output_dir):
-        for filename in filenames:
-            dst = os.path.abspath(f"{root}/{filename}")
-            if dst not in known_targets:
-                logger.info(f"deleting {dst}")
-                os.remove(dst)
-            else:
-                known_targets.remove(dst)
-        for dirname in dirnames:
-            dst = os.path.abspath(f"{root}/{dirname}")
-            if dst not in known_targets:
-                logger.info(f"deleting {dst}")
-                shutil.rmtree(dst)
-            else:
-                known_targets.remove(dst)
-    logger.debug(known_targets)
-
     generate_feed(
         articles,
         args.output_dir,
@@ -310,6 +285,8 @@ def build(args: argparse.Namespace) -> None:
     generate_index(articles, index_template, args.output_dir)
     generate_archive(articles, archive_template, args.output_dir)
     generate_tags(articles, tags_template, tag_template, args.output_dir)
+    generate_search(articles, pages, 'corpus.db')
+    logger.info("Done.")
 
 
 def process_markdown(
@@ -537,5 +514,48 @@ def generate_tags(
             fh.write(result)
 
 
+def generate_search(
+    articles: list[tuple[str, dict[str, Any]]],
+    pages: list[tuple[str, dict[str, Any]]],
+    db: str,
+) -> None:
+    """Generate Search.
+
+    Parameters
+    ----------
+    articles, pages
+    db
+        path to sqlite file
+
+    """
+    logger.info("Generating full text search.")
+    conn = sqlite3.connect(db)
+    with conn:
+        conn.executescript("""
+            drop table if exists corpus;
+
+            create virtual table corpus using fts5(
+                link,
+                text,
+                tokenize = porter
+            );
+        """)
+
+    with conn:
+        for dst, context in articles:
+            text = context['content']
+            conn.execute("""
+                insert into corpus(link, text)
+                values(:link, :text)
+            """, dict(link=dst, text=text))
+
+        for dst, context in pages:
+            text = context['content']
+            conn.execute("""
+                insert into corpus(link, text)
+                values(:link, :text)
+            """, dict(link=dst, text=text))
+
+
 if __name__ == "__main__":
     main()

tests/test_blag.py

@@ -312,22 +312,6 @@ foo bar
     assert os.path.exists(f"{args.output_dir}/tags/bar.html")
 
 
-def test_remove_extra_files(args):
-    """Test that extra files are removed."""
-    # create a file and directory in output dir that have no corresponding
-    # source
-    file_path = f'{args.output_dir}/a'
-    dir_path = f'{args.output_dir}/b'
-    fh = open(file_path, 'w')
-    fh.close()
-    os.mkdir(dir_path)
-
-    blag.build(args)
-
-    assert not os.path.exists(file_path)
-    assert not os.path.exists(dir_path)
-
-
 @pytest.mark.parametrize(
     "template",
     [