build: prevent unnecessary subdir work

improve caching
sort languages for interrun stability

build/fsfe_website_build/build.py

@@ -40,7 +40,7 @@ def parse_arguments() -> argparse.Namespace:
         "--languages",
         help="Languages to build website in.",
         default=[],
-        type=lambda langs: langs.split(","),
+        type=lambda langs: sorted(langs.split(",")),
     )
     parser.add_argument(
         "--log-level",
@@ -116,10 +116,17 @@ def main():
         # otherwise. This symlinks make sure that phase 2 can easily use the right file
         # for each language
         global_symlinks(
-            args.languages
-            if args.languages
-            else list(
-                map(lambda path: path.name, Path(".").glob("global/languages/??"))
+            (
+                args.languages
+                if args.languages
+                else sorted(
+                    list(
+                        map(
+                            lambda path: path.name,
+                            Path(".").glob("global/languages/??"),
+                        )
+                    ),
+                )
             ),
             pool,
         )
@@ -146,11 +153,13 @@ def main():
             languages = (
                 args.languages
                 if args.languages
-                else list(
-                    set(
-                        map(
-                            lambda path: lang_from_filename(path),
-                            site.glob("**/*.*.xhtml"),
+                else sorted(
+                    list(
+                        set(
+                            map(
+                                lambda path: lang_from_filename(path),
+                                site.glob("**/*.*.xhtml"),
+                            )
                         )
                     )
                 )

fsfe.org/events/subdir.py

@@ -7,6 +7,8 @@ import multiprocessing
 from pathlib import Path
 from textwrap import dedent
 
+from build.lib.misc import update_if_changed
+
 logger = logging.getLogger(__name__)
 
 
@@ -19,18 +21,19 @@ def _gen_archive_index(working_dir: Path, languages: list[str], directory: Path)
             logger.debug("Template Exists!")
             content = template.read_text()
             content = content.replace(":YYYY:", directory.name)
-            directory.joinpath(f"index.{lang}.xhtml").write_text(content)
+            update_if_changed(directory.joinpath(f"index.{lang}.xhtml"), content)
 
 
 def _gen_index_sources(directory: Path):
-    directory.joinpath("index.sources").write_text(
+    update_if_changed(
+        directory.joinpath("index.sources"),
         dedent(
             f"""\
                 {directory}/event-*:[]
                 {directory}/.event-*:[]
                 {directory.parent}/.localmenu:[]
             """
-        )
+        ),
     )
 
 

fsfe.org/news/subdir.py

@@ -22,18 +22,19 @@ def _gen_archive_index(working_dir: Path, languages: list[str], directory: Path)
             logger.debug("Template Exists!")
             content = template.read_text()
             content = content.replace(":YYYY:", directory.name)
-            directory.joinpath(f"index.{lang}.xhtml").write_text(content)
+            update_if_changed(directory.joinpath(f"index.{lang}.xhtml"), content)
 
 
 def _gen_index_sources(directory: Path):
-    directory.joinpath("index.sources").write_text(
+    update_if_changed(
+        directory.joinpath("index.sources"),
         dedent(
             f"""\
                 {directory}/news-*:[]
                 {directory}/.news-*:[]
                 {directory.parent}/.localmenu:[]
             """
-        )
+        ),
     )
 
 

status.fsfe.org/filler/early_subdir.py

@@ -14,9 +14,11 @@ from fsfe_website_build.lib.misc import (
 logger = logging.getLogger(__name__)
 
 
-def _create_index(
-    target_file: Path,
-):
+def run(processes: int, working_dir: Path) -> None:
+    """
+    Place filler indices to encourage the site to
+    ensure that status pages for all langs are build.
+    """
     # Create the root element
     page = etree.Element("html")
 
@@ -28,26 +30,19 @@ def _create_index(
     title.text = "filler"
     head = etree.SubElement(page, "body")
 
-    result_str = etree.tostring(page, xml_declaration=True, encoding="utf-8").decode(
+    index_content = etree.tostring(page, xml_declaration=True, encoding="utf-8").decode(
         "utf-8"
     )
-    update_if_changed(target_file, result_str)
-
-
-def run(processes: int, working_dir: Path) -> None:
-    """
-    Place filler indices to encourage the site to
-    ensure that status pages for all langs are build.
-    """
 
     with multiprocessing.Pool(processes) as pool:
-        pool.map(
-            _create_index,
+        pool.starmap(
+            update_if_changed,
             map(
                 lambda path: (
                     working_dir.joinpath(
                         f"index.{path.name}.xhtml",
-                    )
+                    ),
+                    index_content,
                 ),
                 Path().glob("global/languages/*"),
             ),

status.fsfe.org/translations/subdir.py

@@ -93,7 +93,8 @@ def _create_overview(
     data: dict[str : dict[int : list[dict]]],
 ):
     work_file = target_dir.joinpath("langs.en.xml")
-    work_file.parent.mkdir(parents=True, exist_ok=True)
+    if not target_dir.exists():
+        target_dir.mkdir(parents=True)
     # Create the root element
     page = etree.Element("translation-overall-status")
 
@@ -197,41 +198,47 @@ def run(languages: list[str], processes: int, working_dir: Path) -> None:
 
         # Generate our file lists by priority
         # Super hardcoded unfortunately
-        files_by_prio = dict()
-        files_by_prio[1] = list(Path("fsfe.org/").glob("index.en.xhtml")) + list(
+        files_by_priority = dict()
+        files_by_priority[1] = list(Path("fsfe.org/").glob("index.en.xhtml")) + list(
             Path("fsfe.org/freesoftware/").glob("freesoftware.en.xhtml")
         )
-        files_by_prio[2] = list(Path("fsfe.org/activities/").glob("*/activity.en.xml"))
-        files_by_prio[3] = (
+        files_by_priority[2] = list(
+            Path("fsfe.org/activities/").glob("*/activity.en.xml")
+        )
+        files_by_priority[3] = (
             list(Path("fsfe.org/activities/").glob("*.en.xhtml"))
             + list(Path("fsfe.org/activities/").glob("*.en.xml"))
             + list(Path("fsfe.org/freesoftware/").glob("*.en.xhtml"))
             + list(Path("fsfe.org/freesoftware/").glob("*.en.xml"))
         )
-        files_by_prio[4] = (
+        files_by_priority[4] = (
             list(Path("fsfe.org/order/").glob("*.en.xml"))
             + list(Path("fsfe.org/order/").glob("*.en.xhtml"))
             + list(Path("fsfe.org/contribute/").glob("*.en.xml"))
             + list(Path("fsfe.org/contribute/").glob("*.en.xhtml"))
         )
-        files_by_prio[5] = list(Path("fsfe.org/order/").glob("**/*.en.xml")) + list(
+        files_by_priority[5] = list(Path("fsfe.org/order/").glob("**/*.en.xml")) + list(
             Path("fsfe.org/order/").glob("**/*.en.xhtml")
         )
 
-        for priority in sorted(files_by_prio.keys(), reverse=True):
-            files_by_prio[priority] = list(
+        # Make remove files from a priority if they already exist in a higher priority
+        for priority in sorted(files_by_priority.keys(), reverse=True):
+            files_by_priority[priority] = list(
                 filter(
                     lambda path: not any(
-                        [(path in files_by_prio[prio]) for prio in range(1, priority)]
+                        [
+                            (path in files_by_priority[priority])
+                            for priority in range(1, priority)
+                        ]
                     ),
-                    files_by_prio[priority],
+                    files_by_priority[priority],
                 )
             )
 
         files_by_lang_by_prio = {}
         for lang in languages:
             files_by_lang_by_prio[lang] = {}
-            for priority in files_by_prio:
+            for priority in sorted(files_by_priority.keys()):
                 files_by_lang_by_prio[lang][priority] = list(
                     filter(
                         lambda result: result is not None,
@@ -239,7 +246,7 @@ def run(languages: list[str], processes: int, working_dir: Path) -> None:
                             _generate_translation_data,
                             [
                                 (lang, priority, file)
-                                for file in files_by_prio[priority]
+                                for file in files_by_priority[priority]
                             ],
                         ),
                     )