refactor: enable more ruff warnings, and then fix raised issues

build/build.py

@@ -9,16 +9,14 @@ import logging
 import multiprocessing
 import sys
 from pathlib import Path
+from textwrap import dedent
 
 from .lib.misc import lang_from_filename
-
 from .phase0.full import full
 from .phase0.global_symlinks import global_symlinks
 from .phase0.prepare_early_subdirectories import prepare_early_subdirectories
-
 from .phase1.run import phase1_run
 from .phase2.run import phase2_run
-
 from .phase3.serve_websites import serve_websites
 from .phase3.stage_to_target import stage_to_target
 
@@ -42,7 +40,7 @@ def parse_arguments() -> argparse.Namespace:
         "--languages",
         help="Languages to build website in.",
         default=[],
-        type=lambda input: input.split(","),
+        type=lambda langs: langs.split(","),
     )
     parser.add_argument(
         "--log-level",
@@ -66,7 +64,7 @@ def parse_arguments() -> argparse.Namespace:
         "--sites",
         help="What site directories to build",
         default=list(filter(lambda path: path.is_dir(), Path().glob("?*.??*"))),
-        type=lambda input: list(map(lambda site: Path(site), input.split(","))),
+        type=lambda sites: list(map(lambda site: Path(site), sites.split(","))),
     )
     parser.add_argument(
         "--stage",
@@ -75,7 +73,12 @@ def parse_arguments() -> argparse.Namespace:
     )
     parser.add_argument(
         "--target",
-        help="Final dirs for websites to be build to. Can be a single path, or a comma separated list of valid rsync targets. Supports custom rsynx extension for specifying ports for ssh targets, name@host:path?port.",
+        help=dedent("""\
+        Final dirs for websites to be build to.
+        Can be a single path, or a comma separated list of valid rsync targets.
+        Supports custom rsynx extension for specifying ports for ssh targets,
+        name@host:path?port.
+        """),
         type=str,
         default="./output/final",
     )
@@ -132,7 +135,10 @@ def main():
                 logger.critical(f"Site {site} does not exist, exiting")
                 sys.exit(1)
             # Early subdirs
-            # for subdir actions that need to be performed very early in the build process. Do not get access to languages to be built in, and other benefits of being ran later.
+            # for subdir actions that need to be performed
+            # very early in the build process.
+            # Do not get access to languages to be built in,
+            # and other benefits of being ran later.
             prepare_early_subdirectories(
                 site,
                 args.processes,

build/lib/misc.py

@@ -30,11 +30,11 @@ def keys_exists(element: dict, *keys: str) -> bool:
     return True
 
 
-def sort_dict(dict: dict) -> dict:
+def sort_dict(in_dict: dict) -> dict:
     """
     Sort dict by keys
     """
-    return {key: val for key, val in sorted(dict.items(), key=lambda ele: ele[0])}
+    return {key: val for key, val in sorted(in_dict.items(), key=lambda ele: ele[0])}
 
 
 def update_if_changed(path: Path, content: str) -> None:
@@ -76,7 +76,8 @@ def lang_from_filename(file: Path) -> str:
     <name>.XX.<ending>, with xx being the lang code.
     """
     lang = file.with_suffix("").suffix.removeprefix(".")
-    # Lang codes should be the iso 631 2 letter codes, but sometimes we use "nolang" to srop a file being built
+    # Lang codes should be the iso 631 2 letter codes,
+    # but sometimes we use "nolang" to srop a file being built
     if len(lang) != 2 and lang != "nolang":
         logger.critical(
             f"Language {lang} from file {file} not of correct length, exiting"
@@ -91,7 +92,7 @@ def run_command(commands: list) -> str:
         commands,
         capture_output=True,
         # Get output as str instead of bytes
-        universal_newlines=True,
+        text=True,
     )
     if result.returncode != 0:
         logger.critical(f"Command {commands} failed with error")

build/lib/process_file.py

@@ -29,7 +29,8 @@ def _include_xml(file: Path) -> str:
         # to this element instead of the actual content element.
         for elem in root.xpath("version"):
             root.remove(elem)
-        # Iterate over all elements in root node, add a filename attribute and then append the string to work_str
+        # Iterate over all elements in root node, add a filename attribute and
+        # then append the string to work_str
         for elem in root.xpath("*"):
             elem.set("filename", get_basename(file))
             work_str += etree.tostring(elem, encoding="utf-8").decode("utf-8")
@@ -160,43 +161,38 @@ def _build_xmlstream(infile: Path):
     logger.debug(f"action_file: {action_file}")
 
     result_str = f"""
-		<buildinfo
-		  date="{date}"
-		  original="{original_lang}"
-		  filename="/{str(shortname.with_suffix("")).removeprefix("/")}"
-		  fileurl="/{shortname.relative_to(shortname.parts[0]).with_suffix("")}"
-		  dirname="/{shortname.parent}/"
-		  language="{lang}"
-		  translation_state="{translation_state}"
-		>
-
-		<trlist>
-		  {_list_langs(infile)}
-		</trlist>
-
-		<topbanner>
-		{_include_xml(topbanner_xml)}
-		</topbanner>
-		<textsetbackup>
-		{_include_xml(Path("global/data/texts/texts.en.xml"))}
-		</textsetbackup>
-		<textset>
-		{_include_xml(texts_xml)}
-		</textset>
-
-		<document
-		  language="{action_lang}"
-		  {_get_attributes(action_file)}
-		>
-		  <set>
-		    {_auto_sources(action_file, lang)} 
-		  </set>
-
-		  {_include_xml(action_file)}
-		</document>
-
-		</buildinfo>
-        """
+    <buildinfo
+        date="{date}"
+        original="{original_lang}"
+        filename="/{str(shortname.with_suffix("")).removeprefix("/")}"
+        fileurl="/{shortname.relative_to(shortname.parts[0]).with_suffix("")}"
+        dirname="/{shortname.parent}/"
+        language="{lang}"
+        translation_state="{translation_state}"
+    >
+        <trlist>
+            {_list_langs(infile)}
+        </trlist>
+        <topbanner>
+            {_include_xml(topbanner_xml)}
+        </topbanner>
+        <textsetbackup>
+            {_include_xml(Path("global/data/texts/texts.en.xml"))}
+        </textsetbackup>
+        <textset>
+            {_include_xml(texts_xml)}
+        </textset>
+        <document
+            language="{action_lang}"
+            {_get_attributes(action_file)}
+        >
+            <set>
+            {_auto_sources(action_file, lang)}
+            </set>
+            {_include_xml(action_file)}
+        </document>
+    </buildinfo>
+    """
     return result_str
 
 

build/phase0/global_symlinks.py

@@ -10,16 +10,16 @@ from pathlib import Path
 logger = logging.getLogger(__name__)
 
 
-def _do_symlinking(type: str, lang: str) -> None:
+def _do_symlinking(link_type: str, lang: str) -> None:
     """
-    Helper function for doing all of the global symlinking that is suitable for multithreading
+    Helper function for global symlinking that is suitable for multithreading
     """
     target = (
-        Path(f"global/data/{type}/{type}.{lang}.xml")
-        if Path(f"global/data/{type}/{type}.{lang}.xml").exists()
-        else Path(f"global/data/{type}/{type}.en.xml")
+        Path(f"global/data/{link_type}/{link_type}.{lang}.xml")
+        if Path(f"global/data/{link_type}/{link_type}.{lang}.xml").exists()
+        else Path(f"global/data/{link_type}/{link_type}.en.xml")
     )
-    source = Path(f"global/data/{type}/.{type}.{lang}.xml")
+    source = Path(f"global/data/{link_type}/.{link_type}.{lang}.xml")
     if not source.exists():
         source.symlink_to(target.relative_to(source.parent))
 
@@ -35,5 +35,5 @@ def global_symlinks(languages: list[str], pool: multiprocessing.Pool) -> None:
     for each language, also as a prerequisite in the Makefile.
     """
     logger.info("Creating global symlinks")
-    types = ["texts", "topbanner"]
-    pool.starmap(_do_symlinking, product(types, languages))
+    link_types = ["texts", "topbanner"]
+    pool.starmap(_do_symlinking, product(link_types, languages))

build/phase1/index_website.py

@@ -77,7 +77,7 @@ def index_websites(
     logger.info("Creating search indexes")
     # Download all stopwords
     nltkdir = "./.nltk_data"
-    nltk.data.path = [nltkdir] + nltk.data.path
+    nltk.data.path = [nltkdir, *nltk.data.path]
     nltk.download("stopwords", download_dir=nltkdir, quiet=True)
     # Iterate over sites
     if source_dir.joinpath("search/search.js").exists():
@@ -85,8 +85,10 @@ def index_websites(
 
         # Get all xhtml files in languages to be processed
         # Create a list of tuples
-        # The first element of each tuple is the file and the second is a set of stopwords for that language
-        # Use iso639 to get the english name of the language from the two letter iso639-1 code we use to mark files.
+        # The first element of each tuple is the file and
+        # the second is a set of stopwords for that language
+        # Use iso639 to get the english name of the language
+        # from the two letter iso639-1 code we use to mark files.
         # Then if that language has stopwords from nltk, use those stopwords.
         files_with_stopwords = map(
             lambda file: (

build/phase1/run.py

@@ -109,7 +109,8 @@ def phase1_run(
 
     # After this step, the following files will be up to date:
     # * <dir>/.<base>.xmllist for each <dir>/<base>.sources as well as for each
-    #   $site/tags/tagged-<tags>.en.xhtml. These files are used in phase 2 to include the
+    #   $site/tags/tagged-<tags>.en.xhtml.
+    #   These files are used in phase 2 to include the
     #   correct XML files when generating the HTML pages. It is taken care that
     #   these files are only updated whenever their content actually changes, so
     #   they can serve as a prerequisite in the phase 2 Makefile.

build/phase1/update_css.py

@@ -21,16 +21,16 @@ def update_css(
     Then minify it, and place it in the expected location for the build process.
     """
     logger.info("Updating css")
-    dir = source_dir.joinpath("look")
-    if dir.exists():
+    directory = source_dir.joinpath("look")
+    if directory.exists():
         for name in ["fsfe", "valentine"]:
-            if dir.joinpath(name + ".less").exists() and (
-                not dir.joinpath(name + ".min.css").exists()
+            if directory.joinpath(name + ".less").exists() and (
+                not directory.joinpath(name + ".min.css").exists()
                 or any(
                     map(
                         lambda path: path.stat().st_mtime
-                        > dir.joinpath(name + ".min.css").stat().st_mtime,
-                        dir.glob("**/*.less"),
+                        > directory.joinpath(name + ".min.css").stat().st_mtime,
+                        directory.glob("**/*.less"),
                     )
                 )
             ):
@@ -38,10 +38,10 @@ def update_css(
                 result = run_command(
                     [
                         "lessc",
-                        str(dir.joinpath(name + ".less")),
+                        str(directory.joinpath(name + ".less")),
                     ],
                 )
                 update_if_changed(
-                    dir.joinpath(name + ".min.css"),
+                    directory.joinpath(name + ".min.css"),
                     minify.string("text/css", result),
                 )

build/phase1/update_localmenus.py

@@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
 
 
 def _write_localmenus(
-    dir: str, files_by_dir: dict[str, list[Path]], languages: list[str]
+    directory: str, files_by_dir: dict[str, list[Path]], languages: list[str]
 ) -> None:
     """
     Write localmenus for a given directory
@@ -23,11 +23,11 @@ def _write_localmenus(
     base_files = set(
         map(
             lambda filter_file: get_basepath(filter_file),
-            files_by_dir[dir],
+            files_by_dir[directory],
         )
     )
     for lang in languages:
-        file = Path(dir).joinpath(f".localmenu.{lang}.xml")
+        file = Path(directory).joinpath(f".localmenu.{lang}.xml")
         logger.debug(f"Creating {file}")
         page = etree.Element("feed")
 
@@ -92,32 +92,38 @@ def update_localmenus(
     ):
         xslt_root = etree.parse(file)
         if xslt_root.xpath("//localmenu"):
-            dir = xslt_root.xpath("//localmenu/@dir")
-            dir = dir[0] if dir else str(file.parent.relative_to(Path(".")))
-            if dir not in files_by_dir:
-                files_by_dir[dir] = set()
-            files_by_dir[dir].add(file)
-    for dir in files_by_dir:
-        files_by_dir[dir] = sorted(list(files_by_dir[dir]))
+            directory = xslt_root.xpath("//localmenu/@dir")
+            directory = (
+                directory[0] if directory else str(file.parent.relative_to(Path(".")))
+            )
+            if directory not in files_by_dir:
+                files_by_dir[directory] = set()
+            files_by_dir[directory].add(file)
+    for directory in files_by_dir:
+        files_by_dir[directory] = sorted(list(files_by_dir[directory]))
 
     # If any of the source files has been updated, rebuild all .localmenu.*.xml
     dirs = filter(
-        lambda dir: (
+        lambda directory: (
             any(
                 map(
                     lambda file: (
-                        (not Path(dir).joinpath(".localmenu.en.xml").exists())
+                        (not Path(directory).joinpath(".localmenu.en.xml").exists())
                         or (
                             file.stat().st_mtime
-                            > Path(dir).joinpath(".localmenu.en.xml").stat().st_mtime
+                            > Path(directory)
+                            .joinpath(".localmenu.en.xml")
+                            .stat()
+                            .st_mtime
                         )
                     ),
-                    files_by_dir[dir],
+                    files_by_dir[directory],
                 )
             )
         ),
         files_by_dir,
     )
     pool.starmap(
-        _write_localmenus, map(lambda dir: (dir, files_by_dir, languages), dirs)
+        _write_localmenus,
+        map(lambda directory: (directory, files_by_dir, languages), dirs),
     )

build/phase1/update_xmllists.py

@@ -160,8 +160,8 @@ def update_xmllists(
      Creates/update the following files:
 
     * <dir>/.<base>.xmllist for each <dir>/<base>.sources as well as for each
-      fsfe.org/tags/tagged-<tags>.en.xhtml. These files are used in phase 2 to include the
-      correct XML files when generating the HTML pages. It is taken care that
+      fsfe.org/tags/tagged-<tags>.en.xhtml. These files are used in phase 2 to include
+      the correct XML files when generating the HTML pages. It is taken care that
       these files are only updated whenever their content actually changes, so
       they can serve as a prerequisite in the phase 2 Makefile.
 

build/phase2/process_files.py

@@ -49,9 +49,11 @@ def _run_process(
 
 
 def _process_dir(
-    source_dir: Path, languages: list[str], target: Path, dir: Path
+    source_dir: Path, languages: list[str], target: Path, directory: Path
 ) -> None:
-    for basename in set(map(lambda path: path.with_suffix(""), dir.glob("*.??.xhtml"))):
+    for basename in set(
+        map(lambda path: path.with_suffix(""), directory.glob("*.??.xhtml"))
+    ):
         for lang in languages:
             source_file = basename.with_suffix(f".{lang}.xhtml")
             target_file = target.joinpath(
@@ -85,12 +87,13 @@ def process_files(
     Build .html, .rss and .ics files from .xhtml sources
 
     """
-    # TODO for performance it would be better to iterate by processor xls, and parse it only once and pass the xsl object to called function.
+    # TODO for performance it would be better to iterate by processor xls,
+    # and parse it only once and pass the xsl object to called function.
     logger.info("Processing xhtml files")
     pool.starmap(
         _process_dir,
         map(
-            lambda dir: (source_dir, languages, target, dir),
+            lambda directory: (source_dir, languages, target, directory),
             set(map(lambda path: path.parent, source_dir.glob("**/*.*.xhtml"))),
         ),
     )

build/phase3/serve_websites.py

@@ -18,9 +18,9 @@ def _run_webserver(path: str, port: int) -> None:
     serve that dir on that localhost:port for forever.
     """
     os.chdir(path)
-    Handler = http.server.CGIHTTPRequestHandler
+    handler = http.server.CGIHTTPRequestHandler
 
-    with socketserver.TCPServer(("", port), Handler) as httpd:
+    with socketserver.TCPServer(("", port), handler) as httpd:
         httpd.serve_forever()
 
 
@@ -31,9 +31,9 @@ def serve_websites(serve_dir: str, base_port: int, increment_number: int) -> Non
     """
     dirs = sorted(list(filter(lambda path: path.is_dir(), Path(serve_dir).iterdir())))
     serves = []
-    for dir in dirs:
-        port = base_port + (increment_number * dirs.index(dir))
-        logging.info(f"{dir.name} served at http://127.0.0.1:{port}")
-        serves.append((str(dir), port))
+    for index, directory in enumerate(dirs):
+        port = base_port + (increment_number * index)
+        logging.info(f"{directory.name} served at http://127.0.0.1:{port}")
+        serves.append((str(directory), port))
     with multiprocessing.Pool(len(serves)) as pool:
         pool.starmap(_run_webserver, serves)

build/phase3/stage_to_target.py

@@ -21,7 +21,8 @@ def _rsync(stagedir: Path, target: str, port: int) -> None:
             str(stagedir) + "/",
             target,
         ]
-        # Use ssh with a command such that it does not worry about fingerprints, as every connection is a new one basically
+        # Use ssh with a command such that it does not worry about fingerprints,
+        # as every connection is a new one basically
         # Also specify the sshport, and only load this sshconfig if required
         + (
             ["-e", f"ssh -o StrictHostKeyChecking=accept-new -p {port}"]

fsfe.org/events/subdir.py

@@ -10,25 +10,25 @@ from textwrap import dedent
 logger = logging.getLogger(__name__)
 
 
-def _gen_archive_index(working_dir: Path, languages: list[str], dir: Path):
-    logger.debug(f"Operating on dir {dir}")
+def _gen_archive_index(working_dir: Path, languages: list[str], directory: Path):
+    logger.debug(f"Operating on dir {directory}")
     for lang in languages:
         logger.debug(f"Operating on lang {lang}")
         template = working_dir.joinpath(f"archive-template.{lang}.xhtml")
         if template.exists():
             logger.debug("Template Exists!")
             content = template.read_text()
-            content = content.replace(":YYYY:", dir.name)
-            dir.joinpath(f"index.{lang}.xhtml").write_text(content)
+            content = content.replace(":YYYY:", directory.name)
+            directory.joinpath(f"index.{lang}.xhtml").write_text(content)
 
 
-def _gen_index_sources(dir: Path):
-    dir.joinpath("index.sources").write_text(
+def _gen_index_sources(directory: Path):
+    directory.joinpath("index.sources").write_text(
         dedent(
             f"""\
-                {dir}/event-*:[]
-                {dir}/.event-*:[]
-                {dir.parent}/.localmenu:[]
+                {directory}/event-*:[]
+                {directory}/.event-*:[]
+                {directory.parent}/.localmenu:[]
             """
         )
     )
@@ -43,7 +43,7 @@ def run(languages: list[str], processes: int, working_dir: Path) -> None:
         # Copy news archive template to each of the years
         pool.starmap(
             _gen_archive_index,
-            [(working_dir, languages, dir) for dir in years[:-2]],
+            [(working_dir, languages, directory) for directory in years[:-2]],
         )
         logger.debug("Finished Archiving")
         # Generate index.sources for every year

fsfe.org/internal/subdir.py

@@ -5,11 +5,12 @@
 import csv
 import logging
 import os
-import requests
 from pathlib import Path
-import lxml.etree as etree
 from urllib.parse import urlparse
 
+import lxml.etree as etree
+import requests
+
 from build.lib.misc import update_if_changed
 
 logger = logging.getLogger(__name__)

fsfe.org/news/subdir.py

@@ -14,25 +14,25 @@ from build.lib.misc import lang_from_filename, update_if_changed
 logger = logging.getLogger(__name__)
 
 
-def _gen_archive_index(working_dir: Path, languages: list[str], dir: Path):
-    logger.debug(f"Operating on dir {dir}")
+def _gen_archive_index(working_dir: Path, languages: list[str], directory: Path):
+    logger.debug(f"Operating on dir {directory}")
     for lang in languages:
         logger.debug(f"Operating on lang {lang}")
         template = working_dir.joinpath(f"archive-template.{lang}.xhtml")
         if template.exists():
             logger.debug("Template Exists!")
             content = template.read_text()
-            content = content.replace(":YYYY:", dir.name)
-            dir.joinpath(f"index.{lang}.xhtml").write_text(content)
+            content = content.replace(":YYYY:", directory.name)
+            directory.joinpath(f"index.{lang}.xhtml").write_text(content)
 
 
-def _gen_index_sources(dir: Path):
-    dir.joinpath("index.sources").write_text(
+def _gen_index_sources(directory: Path):
+    directory.joinpath("index.sources").write_text(
         dedent(
             f"""\
-                {dir}/news-*:[]
-                {dir}/.news-*:[]
-                {dir.parent}/.localmenu:[]
+                {directory}/news-*:[]
+                {directory}/.news-*:[]
+                {directory.parent}/.localmenu:[]
             """
         )
     )
@@ -40,7 +40,10 @@ def _gen_index_sources(dir: Path):
 
 def _gen_xml_files(working_dir: Path, file: Path):
     logger.debug(f"Transforming {file}")
-    # Would be more efficient to pass this to the function, but this causes a pickling error, and  the faq seems to indicate passing around these objects between threads causes issues
+    # Would be more efficient to pass this to the function,
+    # but this causes a pickling error,
+    # and  the faq seems to indicate passing around these objects
+    # between threads causes issues
     # https://lxml.de/5.0/FAQ.html
     # So I guess we just have to take the performance hit.
     xslt_tree = etree.parse(working_dir.joinpath("xhtml2xml.xsl"))
@@ -66,7 +69,7 @@ def run(languages: list[str], processes: int, working_dir: Path) -> None:
         # Copy news archive template to each of the years
         pool.starmap(
             _gen_archive_index,
-            [(working_dir, languages, dir) for dir in years[:-2]],
+            [(working_dir, languages, directory) for directory in years[:-2]],
         )
         logger.debug("Finished Archiving")
         # Generate index.sources for every year

pyproject.toml

@@ -33,3 +33,17 @@ build-backend = "uv_build"
 module-name = "build"
 module-root = ""
 
+[tool.ruff.lint]
+select = [
+    "A",  # prevent using keywords that clobber python builtins
+    "B",  # bugbear: security warnings
+    "E",  # pycodestyle
+    "F",  # pyflakes
+    "I", # ordered import
+    "ISC",  # implicit string concatenation
+    "N", # pep 8 naming
+    "RUF",  # the ruff developer's own rules
+    "UP",  # alert you when better syntax is available in your python version
+    "W", # pycodestyle warnings
+    "Q", # flake 8 quotes
+ ]

status.fsfe.org/filler/early_subdir.py

@@ -37,7 +37,8 @@ def _create_index(
 
 def run(processes: int, working_dir: Path) -> None:
     """
-    Place filler indices to encourgae the site to ensure that status pages for all langs are build.
+    Place filler indices to encourage the site to
+    ensure that status pages for all langs are build.
     """
 
     with multiprocessing.Pool(processes) as pool:

status.fsfe.org/translations/subdir.py

@@ -149,7 +149,7 @@ def _create_translation_file(
         url=f"https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/{lang_texts_file}",
         filepath=str(lang_texts_file),
     )
-    for missing_text in filter(lambda id: id not in lang_texts, en_texts):
+    for missing_text in filter(lambda text_id: text_id not in lang_texts, en_texts):
         text_elem = etree.SubElement(missing_texts_elem, "text")
         text_elem.text = missing_text
 
@@ -163,7 +163,8 @@ def _create_translation_file(
 
 def run(languages: list[str], processes: int, working_dir: Path) -> None:
     """
-    Build translation-status xmls for languages where the translation status has changed. Xmls are placed in target_dir, and only languages are processed.
+    Build translation-status xmls for languages where the status has changed.
+    Xmls are placed in target_dir, and only languages are processed.
     """
     target_dir = working_dir.joinpath("data/")
     logger.debug(f"Building index of status of translations into dir {target_dir}")
@@ -243,14 +244,10 @@ def run(languages: list[str], processes: int, working_dir: Path) -> None:
         # sadly single treaded, as only one file being operated on
         _create_overview(target_dir, files_by_lang_by_prio)
 
-        for data in [
-            (target_dir, lang, files_by_lang_by_prio[lang])
-            for lang in files_by_lang_by_prio
-        ]:
-            pool.starmap(
-                _create_translation_file,
-                [
-                    (target_dir, lang, files_by_lang_by_prio[lang])
-                    for lang in files_by_lang_by_prio
-                ],
-            )
+        pool.starmap(
+            _create_translation_file,
+            [
+                (target_dir, lang, files_by_lang_by_prio[lang])
+                for lang in files_by_lang_by_prio
+            ],
+        )