feat: minify files on copy, rather than on creation (#5527)

minify more filetypes, like svg and so forth

Co-authored-by: Darragh Elliott <me@delliott.net>
Reviewed-on: #5527
Co-authored-by: delliott <delliott@fsfe.org>
Co-committed-by: delliott <delliott@fsfe.org>

build/fsfe_website_build/phase1/update_css.py

@@ -11,8 +11,6 @@ distributed to the web server.
 import logging
 from typing import TYPE_CHECKING
 
-import minify  # pyright: ignore [reportMissingTypeStubs]
-
 from fsfe_website_build.lib.misc import run_command, update_if_changed
 
 if TYPE_CHECKING:
@@ -33,10 +31,10 @@ def update_css(
     directory = source_dir.joinpath("look")
     if directory.exists():
         for file in directory.glob("main*.less"):
-            minified_path = file.with_suffix(".min.css")
-            if not minified_path.exists() or any(
+            compiled_path = file.with_suffix(".css")
+            if not compiled_path.exists() or any(
                 (
-                    path.stat().st_mtime > minified_path.stat().st_mtime
+                    path.stat().st_mtime > compiled_path.stat().st_mtime
                     for path in directory.glob("**/*.less")
                 ),
             ):
@@ -48,6 +46,6 @@ def update_css(
                     ],
                 )
                 update_if_changed(
-                    minified_path,
-                    minify.string("text/css", result),
+                    compiled_path,
+                    result,
                 )

build/fsfe_website_build/phase2/copy_files.py

@@ -10,6 +10,8 @@ import logging
 import shutil
 from typing import TYPE_CHECKING
 
+import minify  # pyright: ignore [reportMissingTypeStubs]
+
 if TYPE_CHECKING:
     import multiprocessing.pool
     from pathlib import Path
@@ -30,35 +32,75 @@ def _copy_file(target: Path, source_dir: Path, source_file: Path) -> None:
         shutil.copymode(source_file, target_file)
 
 
+def _copy_minify_file(
+    target: Path, source_dir: Path, source_file: Path, mime: str
+) -> None:
+    target_file = target.joinpath(source_file.relative_to(source_dir))
+    if (
+        not target_file.exists()
+        or source_file.stat().st_mtime > target_file.stat().st_mtime
+    ):
+        logger.debug("Copying minified %s to %s", source_file, target_file)
+        target_file.parent.mkdir(parents=True, exist_ok=True)
+        # Do not minify pre minified files
+        if ".min" not in source_file.suffixes:
+            minify.file(mime, str(source_file), str(target_file))
+        else:
+            # Instead just copy them into place
+            target_file.write_bytes(source_file.read_bytes())
+        # preserve file modes
+        shutil.copymode(source_file, target_file)
+
+
 def copy_files(source_dir: Path, pool: multiprocessing.pool.Pool, target: Path) -> None:
     """Copy images, documents etc."""
     logger.info("Copying over media and misc files")
+    # file extensions and mimes of minificable content
+    # excluding files we should not be copying, like xml and html
+    minifiable_content = {
+        ".css": "text/css",
+        ".js": "application/javascript",
+        ".svg": "image/svg+xml",
+    }
+    # Here we copy everything we cannot minify
+
     pool.starmap(
         _copy_file,
         (
             (target, source_dir, file)
-            for file in list(
-                filter(
-                    lambda path: path.is_file()
-                    and path.suffix
-                    not in [
-                        ".md",
-                        ".yml",
-                        ".gitignore",
-                        ".sources",
-                        ".xmllist",
-                        ".xhtml",
-                        ".xsl",
-                        ".xml",
-                        ".less",
-                        ".py",
-                        ".pyc",
-                    ]
-                    and path.name not in ["Makefile"],
-                    source_dir.glob("**/*"),
-                ),
-            )
-            # Special case hard code pass over orde items xml required by cgi script
+            for file in [
+                path
+                for path in source_dir.glob("**/*")
+                if path.is_file()
+                and path.suffix
+                not in
+                # Things we dont want over at all
+                [
+                    ".md",
+                    ".yml",
+                    ".gitignore",
+                    ".sources",
+                    ".xmllist",
+                    ".xhtml",
+                    ".xsl",
+                    ".xml",
+                    ".less",
+                    ".py",
+                    ".pyc",
+                    # and the things we can minify
+                    *minifiable_content.keys(),
+                ]
+                and path.name not in ["Makefile"]
+            ]
+            # Special case hard code pass over order items xml required by cgi script
             + list(source_dir.glob("order/data/items.en.xml"))
         ),
     )
+    # This has to be single threaded as their is an upstream bug that
+    # prevents multihreading the minifier
+    # https://github.com/tdewolff/minify/issues/535
+    for file_suffix, mime in minifiable_content.items():
+        for file in [
+            path for path in source_dir.glob(f"**/*{file_suffix}") if path.is_file()
+        ]:
+            _copy_minify_file(target, source_dir, file, mime)