build: move tags to a subdir instead of a build step (#5273)

it is highly specific to fsfe.org, and so should probably be a subdir instead of a build step

Co-authored-by: Darragh Elliott <me@delliott.net>
Reviewed-on: #5273
Co-authored-by: delliott <delliott@fsfe.org>
Co-committed-by: delliott <delliott@fsfe.org>

build/fsfe_website_build/phase1/run.py

@@ -20,7 +20,6 @@ from .update_css import update_css
 from .update_defaultxsls import update_defaultxsls
 from .update_localmenus import update_localmenus
 from .update_stylesheets import update_stylesheets
-from .update_tags import update_tags
 from .update_xmllists import update_xmllists
 
 logger = logging.getLogger(__name__)
@@ -93,17 +92,6 @@ def phase1_run(
 
     update_localmenus(source_dir, languages, pool)
     # -----------------------------------------------------------------------------
-    # Update tags
-    # -----------------------------------------------------------------------------
-
-    # After this step, the following files will be up to date:
-    # * tags/tagged-<tags>.en.xhtml for each tag used. Apart from being
-    #   automatically created, these are regular source files for HTML pages, and
-    #   in phase 2 are built into pages listing all news items and events for a
-    #   tag.
-    # * tags/.tags.??.xml with a list of the tags used.
-    update_tags(source_dir, languages, pool)
-    # -----------------------------------------------------------------------------
     # Update XML filelists
     # -----------------------------------------------------------------------------
 

fsfe.org/tags/__init__.py

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# __init__.py is a special Python file that allows a directory to become
+# a Python package so it can be accessed using the 'import' statement.

fsfe.org/tags/subdir.py

@@ -7,8 +7,6 @@ import multiprocessing.pool
 from pathlib import Path
 from xml.sax.saxutils import escape
 
-from lxml import etree
-
 from fsfe_website_build.lib.misc import (
     get_basepath,
     keys_exists,
@@ -16,6 +14,7 @@ from fsfe_website_build.lib.misc import (
     sort_dict,
     update_if_changed,
 )
+from lxml import etree
 
 logger = logging.getLogger(__name__)
 
@@ -74,11 +73,7 @@ def _update_tag_sets(
     )
 
 
-def update_tags(
-    source_dir: Path,
-    languages: list[str],
-    pool: multiprocessing.pool.Pool,
-) -> None:
+def run(languages: list[str], processes: int, working_dir: Path) -> None:
     """
     Update Tag pages, xmllists and xmls
 
@@ -97,8 +92,8 @@ def update_tags(
     When a tag has been removed from the last XML file where it has been used,
     the tagged-* are correctly deleted.
     """
-    if source_dir.joinpath("tags").exists():
-        logger.info("Updating tags for %s", source_dir)
+    with multiprocessing.Pool(processes) as pool:
+        logger.debug("Updating tags for %s", working_dir)
         # Create a complete and current map of which tag is used in which files
         files_by_tag = {}
         tags_by_lang = {}
@@ -106,8 +101,8 @@ def update_tags(
         for file in filter(
             lambda file:
             # Not in tags dir of a source_dir
-            source_dir.joinpath("tags") not in file.parents,
-            source_dir.glob("**/*.xml"),
+            working_dir not in file.parents,
+            working_dir.parent.glob("**/*.xml"),
         ):
             for tag in etree.parse(file).xpath("//tag"):
                 # Get the key attribute, and filter out some invalid chars
@@ -146,7 +141,7 @@ def update_tags(
         logger.debug("Updating tag pages")
         pool.starmap(
             _update_tag_pages,
-            ((source_dir, tag, languages) for tag in files_by_tag),
+            ((working_dir.parent, tag, languages) for tag in files_by_tag),
         )
 
         logger.debug("Updating tag lists")
@@ -154,7 +149,7 @@ def update_tags(
             update_if_changed,
             (
                 (
-                    Path(f"{source_dir}/tags/.tagged-{tag}.xmllist"),
+                    Path(f"{working_dir}/.tagged-{tag}.xmllist"),
                     ("\n".join(str(file) for file in files_by_tag[tag]) + "\n"),
                 )
                 for tag in files_by_tag
@@ -178,7 +173,7 @@ def update_tags(
         pool.starmap(
             _update_tag_sets,
             (
-                (source_dir, lang, filecount, files_by_tag, tags_by_lang)
+                (working_dir.parent, lang, filecount, files_by_tag, tags_by_lang)
                 for lang in filter(lambda lang: lang in languages, tags_by_lang.keys())
             ),
         )