feat: ruff python docsting lints (#5444)

And add docstrings so all files are passing.

Co-authored-by: Darragh Elliott <me@delliott.net>
Reviewed-on: #5444
Co-authored-by: delliott <delliott@fsfe.org>
Co-committed-by: delliott <delliott@fsfe.org>

build/fsfe_website_build/__init__.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""The main module for the fsfe website build process."""
+
 from .build import main
 
 __all__ = ["main"]

build/fsfe_website_build/build.py

@@ -1,8 +1,7 @@
-#!/usr/bin/env python3
-
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Command line entrypoint to the build process."""
 
 import argparse
 import logging
@@ -24,10 +23,7 @@ logger = logging.getLogger(__name__)
 
 
 def parse_arguments() -> argparse.Namespace:
-    """
-    Parse the arguments of the website build process
-
-    """
+    """Parse the arguments of the website build process."""
     parser = argparse.ArgumentParser(
         description="Python script to handle building of the fsfe webpage",
     )
@@ -100,9 +96,7 @@ def parse_arguments() -> argparse.Namespace:
 
 
 def main() -> None:
-    """
-    Main process of the website builder
-    """
+    """Parse args and coordinate the website builder."""
     args = parse_arguments()
     logging.basicConfig(
         format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
@@ -117,17 +111,6 @@ def main() -> None:
         # TODO Should also be triggered whenever any build python file is changed
         if args.full:
             full(args.source)
-        # -----------------------------------------------------------------------------
-        # Create XML symlinks
-        # -----------------------------------------------------------------------------
-
-        # After this step, the following symlinks will exist:
-        # * global/data/texts/.texts.<lang>.xml for each language
-        # * global/data/topbanner/.topbanner.<lang>.xml for each language
-        # Each of these symlinks will point to the corresponding file without a dot at
-        # the beginning of the filename, if present, and to the English version
-        # otherwise. This symlinks make sure that phase 2 can easily use the right file
-        # for each language
         global_symlinks(
             args.source,
             (

build/fsfe_website_build/lib/__init__.py

@@ -2,5 +2,11 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Lib functions used across the builder and external consumers.
+
+All files here are held to a higher quality than the rest of the builder.
+They should all be properly typehinted and commented.
+It is considered safe to use them in subdirectory scripts, and CI scripts.
+
+They are the only functions whose interface should not change.
+"""

build/fsfe_website_build/lib/checks.py

@@ -1,6 +1,9 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+
+"""Lib functions used mainly in checks mainly for testing a file."""
+
 import logging
 import sys
 from pathlib import Path
@@ -16,6 +19,7 @@ def compare_files(
     attr_whitelist: set[str] | None = None,
     _path: str = "",
 ) -> list[str]:
+    """Compare two xml files, passes as paths."""
     try:
         t1, t2 = etree.parse(file1), etree.parse(file2)
     except etree.XMLSyntaxError as e:
@@ -31,8 +35,8 @@ def compare_elements(
     attr_whitelist: set[str] | None = None,
     _path: str = "",
 ) -> list[str]:
-    """
-    Recursively compare two XML elements.
+    """Recursively compare two XML elements.
+
     Returns a list of short, informative error strings.
     """
     if attr_whitelist is None:

build/fsfe_website_build/lib/misc.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+#
+"""Misc functions. Mainly common non trivial path manipulations."""
 
 import logging
 import subprocess
@@ -12,9 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 def keys_exists(element: dict, *keys: str) -> bool:
-    """
-    Check if *keys (nested) exists in `element` (dict).
-    """
+    """Check if *keys (nested) exists in `element` (dict)."""
     if not isinstance(element, dict):
         message = "keys_exists() expects dict as first argument."
         logger.error(message)
@@ -30,15 +30,13 @@ def keys_exists(element: dict, *keys: str) -> bool:
 
 
 def sort_dict(in_dict: dict) -> dict:
-    """
-    Sort dict by keys
-    """
+    """Sort dict by keys."""
     return dict(sorted(in_dict.items(), key=lambda ele: ele[0]))
 
 
 def update_if_changed(path: Path, content: str) -> None:
-    """
-    Compare the content of the file at path with the content.
+    """Compare the content of the file at path with the content.
+
     If the file does not exist,
     or its contents does not match content,
     write content to the file.
@@ -49,8 +47,8 @@ def update_if_changed(path: Path, content: str) -> None:
 
 
 def touch_if_newer_dep(file: Path, deps: list[Path]) -> None:
-    """
-    Takes a filepath , and a list of path of its dependencies.
+    """Take a filepath, and a list of path of its dependencies.
+
     If any of the dependencies has been altered more recently than the file,
     touch the file.
 
@@ -62,16 +60,15 @@ def touch_if_newer_dep(file: Path, deps: list[Path]) -> None:
 
 
 def delete_file(file: Path) -> None:
-    """
-    Delete given file using pathlib
-    """
+    """Delete given file using pathlib."""
     logger.debug("Removing file %s", file)
     file.unlink()
 
 
 def lang_from_filename(file: Path) -> str:
-    """
-    Get the lang code from a file, where the filename is of format
+    """Get the lang code from a file.
+
+    Where the filename is of format
     <name>.XX.<ending>, with xx being the lang code.
     """
     lang = file.with_suffix("").suffix.removeprefix(".")
@@ -86,6 +83,11 @@ def lang_from_filename(file: Path) -> str:
 
 
 def run_command(commands: list) -> str:
+    """Run the passed command.
+
+    Useful to standardise how we manage output,
+    and command error handling across the project.
+    """
     try:
         result = subprocess.run(
             commands,
@@ -107,9 +109,7 @@ def run_command(commands: list) -> str:
 
 
 def get_version(file: Path) -> int:
-    """
-    Get the version tag of an xhtml|xml file
-    """
+    """Get the version tag of an xhtml|xml file."""
     xml = etree.parse(file)
     result_list = xml.xpath("/*/version")
     result = result_list[0].text if result_list else str(0)
@@ -118,14 +118,10 @@ def get_version(file: Path) -> int:
 
 
 def get_basepath(file: Path) -> Path:
-    """
-    Return the file with the last two suffixes removed
-    """
+    """Return the file with the last two suffixes removed."""
     return file.with_suffix("").with_suffix("")
 
 
 def get_basename(file: Path) -> str:
-    """
-    Return the name of the file with the last two suffixes removed
-    """
+    """Return the name of the file with the last two suffixes removed."""
     return file.with_suffix("").with_suffix("").name

build/fsfe_website_build/lib/process_file.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Transform a file using a given processor, souring sources and similar correctly."""
+
 import logging
 import re
 from datetime import datetime
@@ -15,9 +17,9 @@ logger = logging.getLogger(__name__)
 
 
 def _get_xmls(file: Path, parser: etree.XMLParser) -> list:
-    """
-    include second level elements of a given XML file
-    this emulates the behaviour of the original
+    """Include second level elements of a given XML file.
+
+    This emulates the behaviour of the original
     build script which wasn't able to load top
     level elements from any file
     """
@@ -37,10 +39,7 @@ def _get_xmls(file: Path, parser: etree.XMLParser) -> list:
 
 
 def _get_attributes(file: Path) -> dict:
-    """
-    get attributes of top level element in a given
-    XHTML file
-    """
+    """Get attributes of top level element in a given XHTML file."""
     tree = etree.parse(file)
     root = tree.getroot()
     attributes = root.items()
@@ -48,8 +47,9 @@ def _get_attributes(file: Path) -> dict:
 
 
 def _get_trlist(source: Path, file: Path) -> etree.Element:
-    """
-    list all languages a file exists in by globbing up
+    """List all languages a file exists.
+
+    Does so by globbing up
     the shortname (i.e. file path with file ending omitted)
     output is readily formatted for inclusion
     in xml stream
@@ -68,9 +68,9 @@ def _get_trlist(source: Path, file: Path) -> etree.Element:
 def _get_set(
     source: Path, action_file: Path, lang: str, parser: etree.XMLParser
 ) -> etree.Element:
-    """
-    import elements from source files, add file name
-    attribute to first element included from each file
+    """Import elements from source files.
+
+    Adds file name attribute to first element included from each file
     """
     doc_set = etree.Element("set")
     list_file = action_file.with_stem(
@@ -110,9 +110,9 @@ def _get_document(
 def _build_xmlstream(
     source: Path, infile: Path, parser: etree.XMLParser
 ) -> etree.Element:
-    """
-    assemble the xml stream for feeding into xsltproc
-    the expected shortname and language flag indicate
+    """Assemble the xml stream for feeding into the transform.
+
+    The expected shortname and language flag indicate
     a single xhtml page to be built
     """
     logger.debug("infile: %s", infile)
@@ -197,9 +197,7 @@ def _build_xmlstream(
 def process_file(
     source: Path, infile: Path, transform: etree.XSLT
 ) -> etree._XSLTResultTree:
-    """
-    Process a given file using the correct xsl sheet
-    """
+    """Process a given file using the correct xsl sheet."""
     logger.debug("Processing %s", infile)
     lang = lang_from_filename(infile)
     parser = etree.XMLParser(remove_blank_text=True, remove_comments=True)

build/fsfe_website_build/phase0/full.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Implementation of the full build logic."""
+
 import logging
 from pathlib import Path
 
@@ -11,9 +13,9 @@ logger = logging.getLogger(__name__)
 
 
 def full(source: Path) -> None:
-    """
-    Git clean the repo to remove all cached artifacts
-    Excluded the root .venv repo, as removing it mid build breaks the build, obviously
+    """Git clean the repo to remove all cached artifacts.
+
+    Excluding the root .venv repo, as removing it mid build breaks the build, obviously.
     """
     logger.info("Performing a full rebuild, git cleaning")
     run_command(

build/fsfe_website_build/phase0/global_symlinks.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Global directory symlinking logic."""
 
 import logging
 import multiprocessing.pool
@@ -11,9 +12,7 @@ logger = logging.getLogger(__name__)
 
 
 def _do_symlinking(source: Path, link_type: str, lang: str) -> None:
-    """
-    Helper function for global symlinking that is suitable for multithreading
-    """
+    """Link a specific file."""
     target = (
         source.joinpath(f"global/data/{link_type}/{link_type}.{lang}.xml")
         if source.joinpath(f"global/data/{link_type}/{link_type}.{lang}.xml").exists()
@@ -27,7 +26,8 @@ def _do_symlinking(source: Path, link_type: str, lang: str) -> None:
 def global_symlinks(
     source: Path, languages: list[str], pool: multiprocessing.pool.Pool
 ) -> None:
-    """
+    """Do all the global symlinking.
+
     After this step, the following symlinks will exist:
     * global/data/texts/.texts.<lang>.xml for each language
     * global/data/topbanner/.topbanner.<lang>.xml for each language

build/fsfe_website_build/phase0/prepare_early_subdirectories.py

@@ -1,6 +1,11 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Prepare subdirectories before we choose languages to build each site in.
+
+Will call the `run` function of any `early_subdir.py`
+found in the website to build source tree.
+"""
 
 import logging
 import sys
@@ -12,9 +17,7 @@ logger = logging.getLogger(__name__)
 def prepare_early_subdirectories(
     source: Path, source_dir: Path, processes: int
 ) -> None:
-    """
-    Find any early subdir scripts in subdirectories and run them
-    """
+    """Find any early subdir scripts in subdirectories and run them."""
     logger.info("Preparing Early Subdirectories for site %s", source_dir)
     for subdir_path in (path.parent for path in source_dir.glob("**/early_subdir.py")):
         logger.info("Preparing early subdirectory %s", subdir_path)

build/fsfe_website_build/phase1/__init__.py

@@ -2,5 +2,11 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Phase 1 logic.
+
+This phase should not touch the target directory.
+
+It should only update the slurce directory with gitignored setup stuff.
+All files should be updated conditioannly on it being changed.
+This ensures proper build caching for stage 2.
+"""

build/fsfe_website_build/phase1/prepare_subdirectories.py

@@ -2,6 +2,12 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Prepare subdirectories, knowing what languages we have already.
+
+Will call the `run` function of any `subdir.py`
+found in the website to build source tree.
+"""
+
 import logging
 import sys
 from pathlib import Path
@@ -15,9 +21,7 @@ def prepare_subdirectories(
     languages: list[str],
     processes: int,
 ) -> None:
-    """
-    Find any subdir scripts in subdirectories and run them
-    """
+    """Find any subdir scripts in subdirectories and run them."""
     logger.info("Preparing Subdirectories")
     for subdir_path in (path.parent for path in source_dir.glob("**/subdir.py")):
         logger.info("Preparing subdirectory %s", subdir_path)

build/fsfe_website_build/phase1/run.py

@@ -1,15 +1,14 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""script for FSFE website build, phase 1.
+
+This script is executed in the root of the source directory tree, and
+creates some .xml and xhtml files as well as some symlinks, all of which
+serve as input files in phase 2. The whole phase 1 runs within the source
+directory tree and does not touch the target directory tree at all.
+"""
 
-# -----------------------------------------------------------------------------
-# script for FSFE website build, phase 1
-# -----------------------------------------------------------------------------
-# This script is executed in the root of the source directory tree, and
-# creates some .xml and xhtml files as well as some symlinks, all of which
-# serve as input files in phase 2. The whole phase 1 runs within the source
-# directory tree and does not touch the target directory tree at all.
-# -----------------------------------------------------------------------------
 import logging
 import multiprocessing.pool
 from pathlib import Path
@@ -31,65 +30,11 @@ def phase1_run(
     processes: int,
     pool: multiprocessing.pool.Pool,
 ) -> None:
-    """
-    Run all the necessary sub functions for phase1.
-    """
+    """Run all the necessary sub functions for phase1."""
     logger.info("Starting Phase 1 - Setup")
-
-    # -----------------------------------------------------------------------------
-    # Update CSS files
-    # -----------------------------------------------------------------------------
-
-    # This step recompiles the less files into the final CSS files to be
-    # distributed to the web server.
     update_css(source_site)
-    # -----------------------------------------------------------------------------
-    # Update XSL stylesheets
-    # -----------------------------------------------------------------------------
-
-    # This step updates (actually: just touches) all XSL files which depend on
-    # another XSL file that has changed since the last build run. The phase 2
-    # Makefile then only has to consider the directly used stylesheet as a
-    # prerequisite for building each file and doesn't have to worry about other
-    # stylesheets imported into that one.
-    # This must run before the "dive into subdirectories" step, because in the news
-    # and events directories, the XSL files, if updated, will be copied for the
-    # per-year archives.
-
     update_stylesheets(source_site, pool)
-    # -----------------------------------------------------------------------------
-    # Dive into subdirectories
-    # -----------------------------------------------------------------------------
-    # Find any makefiles in subdirectories and run them
     prepare_subdirectories(source, source_site, languages, processes)
-
-    # -----------------------------------------------------------------------------
-    # Create XSL symlinks
-    # -----------------------------------------------------------------------------
-
-    # After this step, each directory with source files for HTML pages contains a
-    # symlink named .default.xsl and pointing to the default.xsl "responsible" for
-    # this directory. These symlinks make it easier for the phase 2 Makefile to
-    # determine which XSL script should be used to build a HTML page from a source
-    # file.
-
     update_defaultxsls(source_site, pool)
-    # -----------------------------------------------------------------------------
-    # Update local menus
-    # -----------------------------------------------------------------------------
-
-    # After this step, all .localmenu.??.xml files will be up to date.
-
     update_localmenus(source, source_site, languages, pool)
-    # -----------------------------------------------------------------------------
-    # Update XML filelists
-    # -----------------------------------------------------------------------------
-
-    # After this step, the following files will be up to date:
-    # * <dir>/.<base>.xmllist for each <dir>/<base>.sources as well as for each
-    #   $site/tags/tagged-<tags>.en.xhtml.
-    #   These files are used in phase 2 to include the
-    #   correct XML files when generating the HTML pages. It is taken care that
-    #   these files are only updated whenever their content actually changes, so
-    #   they can serve as a prerequisite in the phase 2 Makefile.
     update_xmllists(source, source_site, languages, pool)

build/fsfe_website_build/phase1/update_css.py

@@ -2,6 +2,12 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Update CSS files.
+
+This step recompiles the less files into the final CSS files to be
+distributed to the web server.
+"""
+
 import logging
 from pathlib import Path
 
@@ -15,8 +21,8 @@ logger = logging.getLogger(__name__)
 def update_css(
     source_dir: Path,
 ) -> None:
-    """
-    If any less files have been changed, update the css.
+    """If any less files have been changed, update the css.
+
     Compile less found at <website>/look/(main*less)
     Then minify it, and place it in the expected location for the build process.
     """

build/fsfe_website_build/phase1/update_defaultxsls.py

@@ -1,6 +1,14 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Create XSL symlinks.
+
+After this step, each directory with source files for HTML pages contains a
+symlink named .default.xsl and pointing to the default.xsl "responsible" for
+this directory. These symlinks make it easier for the phase 2 Makefile to
+determine which XSL script should be used to build a HTML page from a source
+file.
+"""
 
 import logging
 import multiprocessing.pool
@@ -10,9 +18,7 @@ logger = logging.getLogger(__name__)
 
 
 def _do_symlinking(directory: Path) -> None:
-    """
-    In each dir, place a .default.xsl symlink pointing to the nearest default.xsl
-    """
+    """In each dir, place a .default.xsl symlink pointing to the nearest default.xsl."""
     working_dir = directory
     if not directory.joinpath(".default.xsl").exists():
         while not working_dir.joinpath("default.xsl").exists():
@@ -23,9 +29,9 @@ def _do_symlinking(directory: Path) -> None:
 
 
 def update_defaultxsls(source_dir: Path, pool: multiprocessing.pool.Pool) -> None:
-    """
-    Place a .default.xsl into each directory containing source files for
-    HTML pages (*.xhtml). These .default.xsl are symlinks to the first
+    """Place a .default.xsl into each XHTML source directory.
+
+    These .default.xsl are symlinks to the first
     available actual default.xsl found when climbing the directory tree
     upwards, it's the xsl stylesheet to be used for building the HTML
     files from this directory.

build/fsfe_website_build/phase1/update_localmenus.py

@@ -2,6 +2,11 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Update local menus.
+
+After this step, all .localmenu.??.xml files will be up to date.
+"""
+
 import logging
 import multiprocessing.pool
 from pathlib import Path
@@ -18,9 +23,7 @@ def _write_localmenus(
     files_by_dir: dict[str, list[Path]],
     languages: list[str],
 ) -> None:
-    """
-    Write localmenus for a given directory
-    """
+    """Write localmenus for a given directory."""
     # Set of files with no langcode or xhtml extension
     base_files = {get_basepath(filter_file) for filter_file in files_by_dir[directory]}
     for lang in languages:
@@ -81,9 +84,7 @@ def update_localmenus(
     languages: list[str],
     pool: multiprocessing.pool.Pool,
 ) -> None:
-    """
-    Update all the .localmenu.*.xml files containing the local menus.
-    """
+    """Update all the .localmenu.*.xml files containing the local menus."""
     logger.info("Updating local menus")
     # Get a dict of all source files containing local menus
     files_by_dir = {}

build/fsfe_website_build/phase1/update_stylesheets.py

@@ -1,6 +1,17 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Update XSL stylesheets.
+
+This step updates (actually: just touches) all XSL files which depend on
+another XSL file that has changed since the last build run. The phase 2
+build process then only has to consider the directly used stylesheet as a
+prerequisite for building each file and doesn't have to worry about other
+stylesheets imported into that one.
+This must run before the "prepare_subdirectories" step, because in the news
+and events directories, the XSL files, if updated, will be copied for the
+per-year archives.
+"""
 
 import logging
 import multiprocessing.pool
@@ -15,9 +26,7 @@ logger = logging.getLogger(__name__)
 
 
 def _update_sheet(file: Path) -> None:
-    """
-    Update a given xsl file if any of its dependant xsl files have been updated
-    """
+    """Update a given xsl file if any of its dependant xsl files have been updated."""
     xslt_root = etree.parse(file)
     imports = [
         file.parent.joinpath(imp.get("href")).resolve()
@@ -29,13 +38,7 @@ def _update_sheet(file: Path) -> None:
 
 
 def update_stylesheets(source_dir: Path, pool: multiprocessing.pool.Pool) -> None:
-    """
-    This script is called from the phase 1 Makefile and touches all XSL files
-    which depend on another XSL file that has changed since the last build run.
-    The phase 2 Makefile then only has to consider the
-    directly used stylesheet as a prerequisite for building each file and doesn't
-    have to worry about other stylesheets imported into that one.
-    """
+    """Touch all XSL files dependant on an XSL that has changed since last build."""
     logger.info("Updating XSL stylesheets")
     banned = re.compile(r"(\.venv/.*)|(.*\.default\.xsl$)")
     pool.map(

build/fsfe_website_build/phase1/update_xmllists.py

@@ -1,6 +1,16 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Update XML filelists.
+
+After this step, the following files will be up to date:
+* <dir>/.<base>.xmllist for each <dir>/<base>.sources as well as for each
+  $site/tags/tagged-<tags>.en.xhtml.
+  These files are used in phase 2 to include the
+  correct XML files when generating the HTML pages. It is taken care that
+  these files are only updated whenever their content actually changes, so
+  they can serve as a prerequisite in the phase 2 Makefile.
+"""
 
 import datetime
 import fnmatch
@@ -29,9 +39,7 @@ def _update_for_base(  # noqa: PLR0913
     thisyear: str,
     lastyear: str,
 ) -> None:
-    """
-    Update the xmllist for a given base file
-    """
+    """Update the xmllist for a given base file."""
     matching_files = set()
     # If sources exist
     if base.with_suffix(".sources").exists():
@@ -99,9 +107,7 @@ def _update_module_xmllists(
     languages: list[str],
     pool: multiprocessing.pool.Pool,
 ) -> None:
-    """
-    Update .xmllist files for .sources and .xhtml containing <module>s
-    """
+    """Update .xmllist files for .sources and .xhtml containing <module>s."""
     logger.info("Updating XML lists")
     # Get all the bases and stuff before multithreading the update bit
     all_xml = {
@@ -132,9 +138,7 @@ def _update_module_xmllists(
 
 
 def _check_xmllist_deps(source: Path, file: Path) -> None:
-    """
-    If any of the sources in an xmllist are newer than it, touch the xmllist
-    """
+    """If any of the sources in an xmllist are newer than it, touch the xmllist."""
     xmls = set()
     with file.open(mode="r") as fileobj:
         for line in fileobj:
@@ -149,9 +153,7 @@ def _touch_xmllists_with_updated_deps(
     source_dir: Path,
     pool: multiprocessing.pool.Pool,
 ) -> None:
-    """
-    Touch all .xmllist files where one of the contained files has changed
-    """
+    """Touch all .xmllist files where one of the contained files has changed."""
     logger.info("Checking contents of XML lists")
     pool.starmap(
         _check_xmllist_deps,
@@ -165,8 +167,7 @@ def update_xmllists(
     languages: list[str],
     pool: multiprocessing.pool.Pool,
 ) -> None:
-    """
-    Update XML filelists (*.xmllist)
+    """Update XML filelists (*.xmllist).
 
      Creates/update the following files:
 
@@ -175,12 +176,6 @@ def update_xmllists(
       the correct XML files when generating the HTML pages. It is taken care that
       these files are only updated whenever their content actually changes, so
       they can serve as a prerequisite in the phase 2 Makefile.
-
-    Changing or removing tags in XML files is also considered, in which case a
-    file is removed from the .xmllist files.
-
-    When a tag has been removed from the last XML file where it has been used,
-    the tagged-* are correctly deleted.
     """
     _update_module_xmllists(source, source_dir, languages, pool)
     _touch_xmllists_with_updated_deps(source, source_dir, pool)

build/fsfe_website_build/phase2/__init__.py

@@ -2,5 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Website Phase 2.
+
+Should only touch the target directory.
+Transforms the XML/XHTML source files using XSL sheet.
+"""

build/fsfe_website_build/phase2/copy_files.py

@@ -1,6 +1,10 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Copy source files to target directory.
+
+Uses a multithreaded pathlib copy.
+"""
 
 import logging
 import multiprocessing.pool
@@ -24,9 +28,7 @@ def _copy_file(target: Path, source_dir: Path, source_file: Path) -> None:
 
 
 def copy_files(source_dir: Path, pool: multiprocessing.pool.Pool, target: Path) -> None:
-    """
-    Copy images, docments etc
-    """
+    """Copy images, documents etc."""
     logger.info("Copying over media and misc files")
     pool.starmap(
         _copy_file,

build/fsfe_website_build/phase2/create_index_symlinks.py

@@ -1,6 +1,13 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Create index symlinks.
+
+For directories with no existing index,
+where there is a file with the same name as the parent folder,
+EG about/about.en.html
+generate a symlink from about/index.en.html to about.en.html
+"""
 
 import logging
 import multiprocessing.pool
@@ -23,9 +30,7 @@ def create_index_symlinks(
     pool: multiprocessing.pool.Pool,
     target: Path,
 ) -> None:
-    """
-    Create index.* symlinks
-    """
+    """Create index.* symlinks."""
     logger.info("Creating index symlinks")
     pool.map(
         _do_symlinking,

build/fsfe_website_build/phase2/create_language_symlinks.py

@@ -1,6 +1,15 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+#
+"""Generate language symlinks.
+
+Create symlinks from file.<lang>.html to file.html.<lang>.
+
+This is useful as apache multiviews,
+our preferred method of serving by languages,
+takes the file.html.<lang> format.
+"""
 
 import logging
 import multiprocessing.pool
@@ -19,9 +28,7 @@ def create_language_symlinks(
     pool: multiprocessing.pool.Pool,
     target: Path,
 ) -> None:
-    """
-    Create symlinks from file.<lang>.html to file.html.<lang>
-    """
+    """Create symlinks from file.<lang>.html to file.html.<lang>."""
     logger.info("Creating language symlinks")
     pool.map(
         _do_symlinking,

build/fsfe_website_build/phase2/process_files.py

@@ -2,6 +2,12 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Build the lists and pairs file processing.
+
+Takes care to iterate by processor file,
+which is useful to prevent reparsing the XSL multiple times.
+"""
+
 import logging
 import multiprocessing.pool
 from itertools import product
@@ -76,10 +82,7 @@ def process_files(
     pool: multiprocessing.pool.Pool,
     target: Path,
 ) -> None:
-    """
-    Build .html, .rss and .ics files from .xhtml sources
-
-    """
+    """Build .html, .rss and .ics files from .xhtml sources."""
     logger.info("Processing xhtml, rss, ics files")
     # generate a set of unique processing xsls
     xsl_files = {

build/fsfe_website_build/phase2/run.py

@@ -2,9 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# -----------------------------------------------------------------------------
-# script for FSFE website build, phase 2
-# -----------------------------------------------------------------------------
+"""Script for FSFE website build, phase 2."""
+
 import logging
 import multiprocessing.pool
 from pathlib import Path
@@ -24,9 +23,7 @@ def phase2_run(
     pool: multiprocessing.pool.Pool,
     target: Path,
 ) -> None:
-    """
-    Run all the necessary sub functions for phase2.
-    """
+    """Run all the necessary sub functions for phase2."""
     logger.info("Starting Phase 2 - Generating output")
     process_files(source, source_dir, languages, pool, target)
     create_index_symlinks(pool, target)

build/fsfe_website_build/phase3/__init__.py

@@ -2,5 +2,9 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""FSFE website phase 3.
+
+Conditional finishing stuff.
+For example, copy the files from stage to final target(s).
+Or serve the build webpage on localhost.
+"""

build/fsfe_website_build/phase3/serve_websites.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Serve any websites in the output directory on localhost."""
 
 import http.server
 import logging
@@ -16,10 +17,7 @@ logger = logging.getLogger(__name__)
 
 
 def _run_webserver(path: str, port: int) -> None:
-    """
-    Given a path as a string and a port it will
-    serve that dir on that localhost:port for forever.
-    """
+    """Given a path and a port it will serve that dir on that localhost:port."""
     os.chdir(path)
     handler = http.server.CGIHTTPRequestHandler
 
@@ -30,8 +28,10 @@ def _run_webserver(path: str, port: int) -> None:
 def serve_websites(
     serve_dir: Path, sites: list, base_port: int, increment_number: int
 ) -> None:
-    """
-    Takes a target directory, a base port and a number to increment port by per dir
+    """Serve the sites in serve_dir from base port.
+
+    Each sites port is increment by increment_number.
+
     It then serves all directories over http on localhost
     """
     site_names = [site.name for site in sites]

build/fsfe_website_build/phase3/stage_to_target.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Use rsync to copy files to the targets."""
 
 import logging
 import multiprocessing.pool
@@ -35,9 +36,7 @@ def _rsync(stagedir: Path, target: str, port: int) -> None:
 def stage_to_target(
     stagedir: Path, targets: str, pool: multiprocessing.pool.Pool
 ) -> None:
-    """
-    Use a multithreaded rsync to copy the stage dir to all targets.
-    """
+    """Use a multithreaded rsync to copy the stage dir to all targets."""
     logger.info("Rsyncing from stage dir to target dir(s)")
     pool.starmap(
         _rsync,

build/fsfe_website_build_tests/lib/misc_test.py

@@ -1,7 +1,6 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
-
 import subprocess
 from pathlib import Path
 from time import sleep

fsfe.org/events/__init__.py

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Generate the events page and archive."""

fsfe.org/events/subdir.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Generate the events page and archive."""
+
 import logging
 import multiprocessing
 from pathlib import Path
@@ -42,9 +44,7 @@ def _gen_index_sources(source: Path, directory: Path) -> None:
 
 
 def run(source: Path, languages: list[str], processes: int, working_dir: Path) -> None:
-    """
-    preparation for news subdirectory
-    """
+    """Prepare for events subdirectory."""
     with multiprocessing.Pool(processes) as pool:
         years = sorted(working_dir.glob("[0-9][0-9][0-9][0-9]"))
         # Copy news archive template to each of the years

fsfe.org/internal/subdir.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Load internal activities file if possible."""
 
 import csv
 import logging
@@ -16,9 +17,7 @@ logger = logging.getLogger(__name__)
 
 
 def run(source: Path, languages: list[str], processes: int, working_dir: Path) -> None:  # noqa: ARG001 # We allow unused args for subdirs
-    """
-    Internal subdir preparation
-    """
+    """Prepare internal subdirectory."""
     logger.info("Creating activities file")
     raw_url = urlparse(
         "https://git.fsfe.org/FSFE/activities/raw/branch/master/activities.csv",

fsfe.org/news/__init__.py

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Prepare news feed and archive."""

fsfe.org/news/subdir.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Prepare news feed and archive."""
+
 import logging
 import multiprocessing
 from pathlib import Path
@@ -65,9 +67,7 @@ def _gen_xml_files(working_dir: Path, file: Path) -> None:
 
 
 def run(source: Path, languages: list[str], processes: int, working_dir: Path) -> None:
-    """
-    preparation for news subdirectory
-    """
+    """Prepare news subdirectory."""
     with multiprocessing.Pool(processes) as pool:
         years = sorted(working_dir.glob("[0-9][0-9][0-9][0-9]"))
         # Copy news archive template to each of the years

fsfe.org/search/__init__.py

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Generate the search index javascript file."""

fsfe.org/search/subdir.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+"""Create search index javascript file."""
 
 import json
 import logging
@@ -17,8 +18,7 @@ logger = logging.getLogger(__name__)
 
 
 def _find_teaser(document: etree.ElementTree) -> str:
-    """
-    Find a suitable teaser for indexation
+    """Find a suitable teaser for indexation.
 
     Get all the paragraphs in <body> and return the first which contains more
     than 10 words
@@ -34,9 +34,7 @@ def _find_teaser(document: etree.ElementTree) -> str:
 
 
 def _process_file(file: Path, stopwords: set[str]) -> dict:
-    """
-    Generate the search index entry for a given file and set of stopwords
-    """
+    """Generate the search index entry for a given file and set of stopwords."""
     xslt_root = etree.parse(file)
     tags = (
         tag.get("key")
@@ -68,9 +66,10 @@ def _process_file(file: Path, stopwords: set[str]) -> dict:
 
 
 def run(source: Path, languages: list[str], processes: int, working_dir: Path) -> None:  # noqa: ARG001
-    """
-    This step runs a Python tool that creates an index of all news and
-    articles. It extracts titles, teaser, tags, dates and potentially more.
+    """Create a search index.
+
+    Indexes all files, including news and articles.
+    It extracts titles, teaser, tags, dates and potentially more.
     The result will be fed into a JS file.
     """
     # Download all stopwords

fsfe.org/tags/__init__.py

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Generate tag pages for all tags used on the site."""

fsfe.org/tags/subdir.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Generate tag pages for all tags used on the site."""
+
 import logging
 import multiprocessing.pool
 from pathlib import Path
@@ -19,9 +21,7 @@ logger = logging.getLogger(__name__)
 
 
 def _update_tag_pages(site: Path, tag: str, languages: list[str]) -> None:
-    """
-    Update the xhtml pages and xmllists for a given tag
-    """
+    """Update the xhtml pages and xmllists for a given tag."""
     for lang in languages:
         tagfile_source = site.joinpath(f"tags/tagged.{lang}.xhtml")
         if tagfile_source.exists():
@@ -37,9 +37,7 @@ def _update_tag_sets(
     files_by_tag: dict[str, list[Path]],
     tags_by_lang: dict[str, dict[str, str]],
 ) -> None:
-    """
-    Update the .tags.??.xml tagset xmls for a given tag
-    """
+    """Update the .tags.??.xml tagset xmls for a given tag."""
     # Add uout toplevel element
     page = etree.Element("tagset")
 
@@ -73,8 +71,7 @@ def _update_tag_sets(
 
 
 def run(source: Path, languages: list[str], processes: int, working_dir: Path) -> None:  # noqa: ARG001
-    """
-    Update Tag pages, xmllists and xmls
+    """Update Tag pages, xmllists and xmls.
 
      Creates/update the following files:
 
@@ -84,12 +81,6 @@ def run(source: Path, languages: list[str], processes: int, working_dir: Path) -
       tag.
 
     * */tags/.tags.??.xml with a list of the tags used.
-
-    Changing or removing tags in XML files is also considered, in which case a
-    file is removed from the .xmllist files.
-
-    When a tag has been removed from the last XML file where it has been used,
-    the tagged-* are correctly deleted.
     """
     with multiprocessing.Pool(processes) as pool:
         logger.debug("Updating tags for %s", working_dir)

pyproject.toml

@@ -43,6 +43,7 @@ select = [
   "ASYNC", # Flake 8 aysnc
   "B",     # bugbear: security warnings
   "C4",    # Comprehensions
+  "D",     # pydocstyle, recommended with pydoclint
   "E",     # pycodestyle
   "ERA",   # Commented out code
   "F",     # pyflakes
@@ -68,7 +69,14 @@ select = [
   "UP",    # Update syntax to newer versions
   "W",     # pycodestyle warnings
 ]
-
+ignore = [
+  "D203", # incorrect-blank-line-before-class, Conflicts with D211
+  "D213", # multi-line-summary-second-line, Conflicts with D212
+]
+[tool.ruff.lint.per-file-ignores]
+"build/fsfe_website_build_tests/*" = [
+  "D",
+] # We do not need to document the tests.
 [tool.pytest.ini_options]
 testpaths = ["build/fsfe_website_build_tests"]
 python_files = ["*_test.py"]

status.fsfe.org/filler/__init__.py

@@ -2,5 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Fill the status directory with dummy files.
+
+By placing a dummy file for every lang code
+we ensure translation pages for all langs are built.
+"""

status.fsfe.org/filler/early_subdir.py

@@ -2,6 +2,12 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Fill the status directory with dummy files.
+
+By placing a dummy file for every lang code
+we ensure translation pages for all langs are built.
+"""
+
 import logging
 import multiprocessing
 from pathlib import Path
@@ -15,9 +21,9 @@ logger = logging.getLogger(__name__)
 
 
 def run(source: Path, processes: int, working_dir: Path) -> None:
-    """
-    Place filler indices to encourage the site to
-    ensure that status pages for all langs are build.
+    """Place filler indices to encourage the site.
+
+    This ensures that status pages for all langs are build.
     """
     # Create the root element
     page = etree.Element("html")

status.fsfe.org/style_attributes/__init__.py

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Show what files have improper style attributes."""

status.fsfe.org/style_attributes/subdir.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Show what files have improper style attributes."""
+
 import logging
 import multiprocessing
 from collections import defaultdict
@@ -34,9 +36,10 @@ def _worker(path: Path) -> tuple[str, Path, Path, list[tuple[str, str]]] | None:
 
 
 def run(source: Path, languages: list[str], processes: int, working_dir: Path) -> None:  # noqa: ARG001
-    """
-    Generate an XML index that contains every tracked file which has at
-    least one element carrying a style attribute.
+    """Generate an XML index of files with styles.
+
+    This contains every tracked file which has
+    at least one element carrying a style attribute.
     """
     target_dir = working_dir / "data"
     target_dir.mkdir(parents=True, exist_ok=True)

status.fsfe.org/translations/__init__.py

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Generate the translation status of all files/texts."""

status.fsfe.org/translations/subdir.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Generate the translation status of all files/texts."""
+
 import datetime
 import logging
 import multiprocessing
@@ -155,8 +157,8 @@ def _create_translation_file(
 
 
 def run(source: Path, languages: list[str], processes: int, working_dir: Path) -> None:  # noqa: ARG001
-    """
-    Build translation-status xmls for languages where the status has changed.
+    """Build translation-status xmls for languages where the status has changed.
+
     Xmls are placed in target_dir, and only languages are processed.
     """
     target_dir = working_dir.joinpath("data/")

status.fsfe.org/xml_structure/__init__.py

@@ -2,5 +2,4 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-# __init__.py is a special Python file that allows a directory to become
-# a Python package so it can be accessed using the 'import' statement.
+"""Show what files have mismatched xml structures across languages."""

status.fsfe.org/xml_structure/subdir.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+"""Show what files have mismatched xml structures across languages."""
+
 import logging
 import multiprocessing
 from collections import defaultdict
@@ -31,8 +33,8 @@ def _job(
 
 
 def run(source: Path, languages: list[str], processes: int, working_dir: Path) -> None:  # noqa: ARG001
-    """
-    Build xml-structure log for displaying on a status page
+    """Build xml-structure log for displaying on a status page.
+
     Xmls are placed in target_dir, and only passed languages are processed.
     """
     target_dir = working_dir.joinpath("data/")

tools/ci-checks/compare-xml-structure.py

@@ -3,6 +3,9 @@
 # SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
+#
+"""Check that the passed files match xml structure across languages."""
+
 import argparse
 import logging
 import multiprocessing
@@ -34,6 +37,7 @@ def _job(master: Path, other: Path, whitelist: set[str]) -> str | None:
 
 
 def main() -> None:
+    """Check that the passed files match xml structure across languages."""
     parser = argparse.ArgumentParser(
         description="Compare XML structure and attributes. "
         "Use --multi for space-separated list + parallel compares."