feat/python-rewrite (#4762)
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
Rewrite the whole build process in python, for superior speed and maintenance Co-authored-by: Darragh Elliott <me@delliott.xyz> Co-authored-by: Sofía Aritz <sofiaritz@fsfe.org> Co-authored-by: tobiasd <tobiasd@fsfe.org> Co-authored-by: Tobias Diekershoff <tobiasd@fsfe.org> Reviewed-on: #4762 Co-authored-by: delliott <delliott@fsfe.org> Co-committed-by: delliott <delliott@fsfe.org>
This commit is contained in:
parent
56465aa86c
commit
4468cf2337
74
.drone.yml
74
.drone.yml
@ -7,12 +7,16 @@ clone:
|
||||
depth: 150
|
||||
|
||||
steps:
|
||||
- name: checks
|
||||
- name: check-python
|
||||
image: ghcr.io/astral-sh/ruff:latest
|
||||
command: [ "check", "." ]
|
||||
|
||||
- name: check-custom
|
||||
image: debian:bookworm
|
||||
commands:
|
||||
- apt update
|
||||
# Install required packages
|
||||
- apt install --yes coreutils sed grep libxml2-utils git findutils perl-base file mediainfo curl
|
||||
- apt install --yes --no-install-recommends coreutils sed grep libxml2-utils git findutils perl-base file mediainfo curl
|
||||
# Check whether non-EN news item would appear on front-page
|
||||
- bash tools/check-non-en-frontpage.sh news
|
||||
# Run pre-commit checks
|
||||
@ -20,6 +24,61 @@ steps:
|
||||
# Check syntax for all files as a safety net
|
||||
- find . -type f \( -iname "*.xhtml" -o -iname "*.xml" -o -iname "*.xsl" \) -exec xmllint --noout {} +
|
||||
|
||||
- name: deploy-master
|
||||
image: docker:27.4.1
|
||||
environment:
|
||||
# Environment variables necessary for rootless Docker
|
||||
XDG_RUNTIME_DIR: "/run/user/1001"
|
||||
DOCKER_HOST: "unix:///run/user/1001/docker.sock"
|
||||
# Target bunsen directly, and use ipv4 proxies for noddack and gahn, as ipv6 broken.
|
||||
TARGET: "www@bunsen.fsfeurope.org:fsfe.org/global/,www@proxy.noris.fsfeurope.org:fsfe.org/global/?10322,www@proxy.plutex.fsfeurope.org:fsfe.org/global/?10322"
|
||||
KEY_PRIVATE:
|
||||
from_secret: KEY_PRIVATE
|
||||
KEY_PASSWORD:
|
||||
from_secret: KEY_PASSWORD
|
||||
GIT_TOKEN:
|
||||
from_secret: BUILD_TOKEN
|
||||
volumes:
|
||||
# Mounting Docker socket of rootless docker user
|
||||
- name: dockersock
|
||||
path: /run/user/1001/docker.sock
|
||||
commands:
|
||||
- docker ps && echo "tampered with"
|
||||
- docker compose -p fsfe-website run --remove-orphans --build build --target "$TARGET"
|
||||
when:
|
||||
branch:
|
||||
- master
|
||||
event:
|
||||
exclude:
|
||||
- pull_request
|
||||
|
||||
- name: deploy-test
|
||||
image: docker:27.4.1
|
||||
environment:
|
||||
# Environment variables necessary for rootless Docker
|
||||
XDG_RUNTIME_DIR: "/run/user/1001"
|
||||
DOCKER_HOST: "unix:///run/user/1001/docker.sock"
|
||||
# Target bunsen directly, and use ipv4 proxies for noddack and gahn, as ipv6 broken.
|
||||
TARGET: "www@bunsen.fsfeurope.org:test.fsfe.org/global/,www@proxy.noris.fsfeurope.org:test.fsfe.org/global/?10322,www@proxy.plutex.fsfeurope.org:test.fsfe.org/global/?10322"
|
||||
KEY_PRIVATE:
|
||||
from_secret: KEY_PRIVATE
|
||||
KEY_PASSWORD:
|
||||
from_secret: KEY_PASSWORD
|
||||
GIT_TOKEN:
|
||||
from_secret: BUILD_TOKEN
|
||||
volumes:
|
||||
# Mounting Docker socket of rootless docker user
|
||||
- name: dockersock
|
||||
path: /run/user/1001/docker.sock
|
||||
commands:
|
||||
- docker ps && echo "tampered with"
|
||||
- docker compose -p fsfe-website run --remove-orphans --build build --target "$TARGET"
|
||||
when:
|
||||
branch:
|
||||
- test
|
||||
event:
|
||||
exclude:
|
||||
- pull_request
|
||||
trigger:
|
||||
branch:
|
||||
- master
|
||||
@ -27,8 +86,17 @@ trigger:
|
||||
event:
|
||||
- push
|
||||
- pull_request
|
||||
|
||||
node:
|
||||
cont2: noris
|
||||
|
||||
volumes:
|
||||
# Define Docker socket of rootless docker user
|
||||
- name: dockersock
|
||||
host:
|
||||
path: /run/user/1001/docker.sock
|
||||
---
|
||||
kind: signature
|
||||
hmac: 4c0dd0f272458d12234c72f66c4d420069591cac83819644df3c03a280102ded
|
||||
hmac: 3603190ca39d5c40ceb70998239a593bb47d95610608495c72f486e9d3d53788
|
||||
|
||||
...
|
||||
|
||||
13
.gitignore
vendored
13
.gitignore
vendored
@ -13,11 +13,22 @@ global/data/topbanner/.topbanner.??.xml
|
||||
.localmenu.*.xml
|
||||
.*.xmllist
|
||||
fsfe.org/search/index.js
|
||||
fsfe.org/tags/tagged-*.en.xhtml
|
||||
fsfe.org/tags/tagged-*.??.xhtml
|
||||
fsfe.org/tags/.tags.??.xml
|
||||
global/data/modules/fsfe-activities-options.en.xml
|
||||
# Local build stuff
|
||||
output
|
||||
# Python venv
|
||||
.venv
|
||||
__pycache__
|
||||
#Nltk
|
||||
.nltk_data
|
||||
|
||||
## Status dir stuff
|
||||
status.fsfe.org/*/data*/*
|
||||
|
||||
# Secrets
|
||||
# docker compose
|
||||
.env
|
||||
# drone
|
||||
secrets.txt
|
||||
|
||||
47
Dockerfile
47
Dockerfile
@ -1,23 +1,34 @@
|
||||
FROM debian:bookworm-slim
|
||||
FROM debian:latest
|
||||
|
||||
# Install deps
|
||||
RUN apt update
|
||||
# Install required packages
|
||||
RUN apt install --yes \
|
||||
bash \
|
||||
coreutils \
|
||||
RUN apt install --yes --no-install-recommends \
|
||||
rsync \
|
||||
xsltproc \
|
||||
libxml2-utils \
|
||||
sed \
|
||||
findutils \
|
||||
grep \
|
||||
make \
|
||||
libc-bin \
|
||||
wget \
|
||||
procps \
|
||||
libxslt1.1 \
|
||||
libxml2 \
|
||||
golang \
|
||||
python3 \
|
||||
python3-bs4
|
||||
|
||||
WORKDIR /fsfe-websites
|
||||
ENTRYPOINT ["bash", "./build.sh" ]
|
||||
python3-venv \
|
||||
python3-pip \
|
||||
git \
|
||||
node-less \
|
||||
openssh-client \
|
||||
expect
|
||||
|
||||
|
||||
# Setup venv
|
||||
ENV VIRTUAL_ENV=/opt/venv
|
||||
RUN python3 -m venv $VIRTUAL_ENV
|
||||
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
# Copy the requirements
|
||||
# Done in a seperate step for optimal docker caching
|
||||
COPY ./requirements.txt /website-source/requirements.txt
|
||||
RUN pip install -r /website-source/requirements.txt
|
||||
|
||||
# Copy everything else
|
||||
COPY . /website-source/
|
||||
WORKDIR /website-source
|
||||
|
||||
ENTRYPOINT [ "bash", "./entrypoint.sh" ]
|
||||
|
||||
|
||||
|
||||
157
Makefile
157
Makefile
@ -1,157 +0,0 @@
|
||||
# -----------------------------------------------------------------------------
|
||||
# Makefile for FSFE website build, phase 1
|
||||
# -----------------------------------------------------------------------------
|
||||
# This Makefile is executed in the root of the source directory tree, and
|
||||
# creates some .xml and xhtml files as well as some symlinks, all of which
|
||||
# serve as input files in phase 2. The whole phase 1 runs within the source
|
||||
# directory tree and does not touch the target directory tree at all.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
.PHONY: all .FORCE
|
||||
.FORCE:
|
||||
|
||||
# This will be overwritten in the command line running this Makefile.
|
||||
build_env = development
|
||||
languages = none
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Build search index
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# This step runs a Python tool that creates an index of all news and
|
||||
# articles. It extracts titles, teaser, tags, dates and potentially more.
|
||||
# The result will be fed into a JS file.
|
||||
|
||||
.PHONY: searchindex
|
||||
all: searchindex
|
||||
searchindex:
|
||||
python3 tools/index-website.py
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update CSS files
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# This step recompiles the less files into the final CSS files to be
|
||||
# distributed to the web server.
|
||||
|
||||
ifneq ($(build_env),development)
|
||||
websites:=$(shell find . -mindepth 2 -maxdepth 2 -type d -regex "./[a-z\.]+\.[a-z]+/look")
|
||||
all: $(foreach dir,$(websites), $(dir)/fsfe.min.css $(dir)/valentine.min.css)
|
||||
$(dir $@)%.min.css: $(shell find $(dir $@) -name '*.less')
|
||||
echo "* Compiling $@"
|
||||
lessc "$*.less" -x "$@"
|
||||
endif
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update XSL stylesheets
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# This step updates (actually: just touches) all XSL files which depend on
|
||||
# another XSL file that has changed since the last build run. The phase 2
|
||||
# Makefile then only has to consider the directly used stylesheet as a
|
||||
# prerequisite for building each file and doesn't have to worry about other
|
||||
# stylesheets imported into that one.
|
||||
# This must run before the "dive into subdirectories" step, because in the news
|
||||
# and events directories, the XSL files, if updated, will be copied for the
|
||||
# per-year archives.
|
||||
|
||||
.PHONY: stylesheets
|
||||
all: stylesheets
|
||||
stylesheets: $(SUBDIRS)
|
||||
tools/update_stylesheets.sh
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Dive into subdirectories
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
SUBDIRS := $(shell find . -regex "./[a-z\.]+\.[a-z]+/.*/Makefile" | xargs dirname)
|
||||
|
||||
all: $(SUBDIRS)
|
||||
$(SUBDIRS): .FORCE
|
||||
echo "* Preparing subdirectory $@"
|
||||
$(MAKE) --silent --directory=$@ languages="$(languages)"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create XML symlinks
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, the following symlinks will exist:
|
||||
# * global/data/texts/.texts.<lang>.xml for each language
|
||||
# * global/data/topbanner/.topbanner.<lang>.xml for each language
|
||||
# Each of these symlinks will point to the corresponding file without a dot at
|
||||
# the beginning of the filename, if present, and to the English version
|
||||
# otherwise. This symlinks make sure that phase 2 can easily use the right file
|
||||
# for each language, also as a prerequisite in the Makefile.
|
||||
|
||||
TEXTS_LINKS := $(foreach lang,$(languages),global/data/texts/.texts.$(lang).xml)
|
||||
|
||||
all: $(TEXTS_LINKS)
|
||||
global/data/texts/.texts.%.xml: .FORCE
|
||||
if [ -f global/data/texts/texts.$*.xml ]; then \
|
||||
ln -sf texts.$*.xml $@; \
|
||||
else \
|
||||
ln -sf texts.en.xml $@; \
|
||||
fi
|
||||
|
||||
TOPBANNER_LINKS := $(foreach lang,$(languages),global/data/topbanner/.topbanner.$(lang).xml)
|
||||
|
||||
all: $(TOPBANNER_LINKS)
|
||||
global/data/topbanner/.topbanner.%.xml: .FORCE
|
||||
if [ -f global/data/topbanner/topbanner.$*.xml ]; then \
|
||||
ln -sf topbanner.$*.xml $@; \
|
||||
else \
|
||||
ln -sf topbanner.en.xml $@; \
|
||||
fi
|
||||
|
||||
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
# The following steps are handled in an external script, because the list of
|
||||
# files to generate is not known when the Makefile starts - some new tags might
|
||||
# be introduced when generating the .xml files in the news/* subdirectories.
|
||||
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create XSL symlinks
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, each directory with source files for HTML pages contains a
|
||||
# symlink named .default.xsl and pointing to the default.xsl "responsible" for
|
||||
# this directory. These symlinks make it easier for the phase 2 Makefile to
|
||||
# determine which XSL script should be used to build a HTML page from a source
|
||||
# file.
|
||||
|
||||
.PHONY: default_xsl
|
||||
all: default_xsl
|
||||
default_xsl:
|
||||
tools/update_defaultxsls.sh
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update local menus
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, all .localmenu.??.xml files will be up to date.
|
||||
|
||||
.PHONY: localmenus
|
||||
all: localmenus
|
||||
localmenus: $(SUBDIRS)
|
||||
tools/update_localmenus.sh "$(languages)"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update XML filelists
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, the following files will be up to date:
|
||||
# * tags/tagged-<tags>.en.xhtml for each tag used. Apart from being
|
||||
# automatically created, these are regular source files for HTML pages, and
|
||||
# in phase 2 are built into pages listing all news items and events for a
|
||||
# tag.
|
||||
# * tags/.tags.??.xml with a list of the tags useed.
|
||||
# * <dir>/.<base>.xmllist for each <dir>/<base>.sources as well as for each
|
||||
# tags/tagged-<tags>.en.xhtml. These files are used in phase 2 to include the
|
||||
# correct XML files when generating the HTML pages. It is taken care that
|
||||
# these files are only updated whenever their content actually changes, so
|
||||
# they can serve as a prerequisite in the phase 2 Makefile.
|
||||
|
||||
.PHONY: xmllists
|
||||
all: xmllists
|
||||
xmllists: $(SUBDIRS)
|
||||
tools/update_xmllists.sh "$(languages)"
|
||||
93
README.md
93
README.md
@ -5,11 +5,11 @@ This repository contains the source files of [fsfe.org](https://fsfe.org), pdfre
|
||||
|
||||
## Table of Contents
|
||||
|
||||
* [Technical information](#technical-information)
|
||||
* [Structure](#structure)
|
||||
* [Contribute](#contribute)
|
||||
* [Translate](#translate)
|
||||
* [Build](#build)
|
||||
- [Technical information](#technical-information)
|
||||
- [Structure](#structure)
|
||||
- [Contribute](#contribute)
|
||||
- [Translate](#translate)
|
||||
- [Build](#build)
|
||||
|
||||
## Technical information
|
||||
|
||||
@ -19,7 +19,7 @@ For information on how the build process works see [docs subfolder](./docs/overv
|
||||
|
||||
## Structure
|
||||
|
||||
Most files are XHTML files organised in a rather logical folder structure.
|
||||
Most files are XHTML files organised in a rather logical folder structure.
|
||||
|
||||
Every website served using this repo has its own folder with the full domain name it is to be served from.
|
||||
|
||||
@ -27,42 +27,43 @@ Every website served using this repo has its own folder with the full domain nam
|
||||
|
||||
This repository also contains the source files of other websites the FSFE hosts:
|
||||
|
||||
* `fsfe.org` for [fsfe.org](http://fsfe.org)
|
||||
* `activities/android` for [freeyourandroid.org](http://freeyourandroid.org)
|
||||
* `activities/ilovefs` for [ilovefs.org](http://ilovefs.org)
|
||||
* `drm.info` for [drm.info](http://drm.info)
|
||||
* `pdfreaders.org` for [pdfreaders.org](http://pdfreaders.org)
|
||||
* [test.fsfe.org](https://test.fsfe.org) is fsfe.org built from the test branch of this repository
|
||||
- `fsfe.org` for [fsfe.org](http://fsfe.org)
|
||||
- `activities/android` for [freeyourandroid.org](http://freeyourandroid.org)
|
||||
- `activities/ilovefs` for [ilovefs.org](http://ilovefs.org)
|
||||
- `drm.info` for [drm.info](http://drm.info)
|
||||
- `pdfreaders.org` for [pdfreaders.org](http://pdfreaders.org)
|
||||
- [test.fsfe.org](https://test.fsfe.org) is fsfe.org built from the test branch of this repository
|
||||
|
||||
### Important folders
|
||||
|
||||
Notable top level directories are:
|
||||
|
||||
* `build`: Mostly custom Bash and XSL scripts to build the website
|
||||
* `global`: Globally used data files and modules, also the static translated strings.
|
||||
* `tools`: Contains miscellaneous XML, XSL, and SH files.
|
||||
- `build`: Mostly custom Bash and XSL scripts to build the website
|
||||
- `global`: Globally used data files and modules, also the static translated strings.
|
||||
- `tools`: Contains miscellaneous XML, XSL, and SH files.
|
||||
|
||||
And of course the different website folders.
|
||||
|
||||
And here are dome notable directories inside the folder for the main webpage, fsfe.org.
|
||||
* `about`: Information about the FSFE itself, its team members etc
|
||||
* `activities`: All specific FSFE activities
|
||||
* `at`, `de`, `ee` etc: Folders used for the FSFE country teams
|
||||
* `cgi-bin`: Our very few CGI scripts
|
||||
* `error`: Custom 4xx and 5xx error pages
|
||||
* `events`: Files for our events, ordered by year
|
||||
* `freesoftware`: More timeless pages explaining Free Software and related topics
|
||||
* `graphics`: Icons, pictures and logos
|
||||
* `internal`: Forms used mostly by FSFE staff for internal processes
|
||||
* `look`: CSS and other style files
|
||||
* `news`: Files for news articles, press releases, and newsletters ordered by year
|
||||
* `order`: Our web shop
|
||||
* `scripts`: JavaScript files used on our pages
|
||||
* `tags`: Files necessary to display used tags throughout the website. Mostly automatically generated
|
||||
|
||||
- `about`: Information about the FSFE itself, its team members etc
|
||||
- `activities`: All specific FSFE activities
|
||||
- `at`, `de`, `ee` etc: Folders used for the FSFE country teams
|
||||
- `cgi-bin`: Our very few CGI scripts
|
||||
- `error`: Custom 4xx and 5xx error pages
|
||||
- `events`: Files for our events, ordered by year
|
||||
- `freesoftware`: More timeless pages explaining Free Software and related topics
|
||||
- `graphics`: Icons, pictures and logos
|
||||
- `internal`: Forms used mostly by FSFE staff for internal processes
|
||||
- `look`: CSS and other style files
|
||||
- `news`: Files for news articles, press releases, and newsletters ordered by year
|
||||
- `order`: Our web shop
|
||||
- `scripts`: JavaScript files used on our pages
|
||||
- `tags`: Files necessary to display used tags throughout the website. Mostly automatically generated
|
||||
|
||||
## Contribute
|
||||
|
||||
Become member of our awesome [webmaster team](https://fsfe.org/contribute/web/) and help to improve our online information platform!
|
||||
Become member of our awesome [webmaster team](https://fsfe.org/contribute/web/) and help to improve our online information platform!
|
||||
|
||||
## Translate
|
||||
|
||||
@ -79,30 +80,40 @@ There are two ways to build and develop the directory locally. Initial builds of
|
||||
Alterations to build scripts or the files used site-wide will result in near full rebuilds.
|
||||
|
||||
### Native
|
||||
We can either install the required dependencies manually using our preferred package manager. If you are a nix use one can run `nix-shell` to enter a shell with the required build dependencies.
|
||||
|
||||
The required binary names are
|
||||
We can either install the required dependencies manually using our preferred package manager. If you are a nix use one can run `nix-shell` to enter a shell with the required build dependencies, with the python `virtualenv` already installed and activated.
|
||||
|
||||
If installing manually, the required binary names are
|
||||
|
||||
```
|
||||
realpath rsync xsltproc xmllint sed find egrep grep wc make tee date iconv wget shuf python3
|
||||
python3 pip
|
||||
```
|
||||
The package names for Debian, are
|
||||
|
||||
Also needed are the libraries
|
||||
|
||||
```
|
||||
bash bash-completion coreutils diffutils findutils inotify-tools libxml2-utils libxslt make procps python3 rsync
|
||||
libxml2 libxslt
|
||||
```
|
||||
|
||||
Then, we must activate a Python virtual env and install the python dependencies.
|
||||
|
||||
```
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
After getting the dependencies one way or another we can actually build and serve the pages.
|
||||
|
||||
The pages can be built and served by running `./build.sh`. Try `--help` for more information. The simple web server used lacks the features of `apache` which used on the FSFE web servers. This is why no index is automatically selected form and directory and other behaviors.
|
||||
The pages can be built and served by running `./build.py`. Try `--help` for more information. The simple web server used lacks the features of `apache` which used on the FSFE web servers. This is why no index is automatically selected for each directory and other behaviours.
|
||||
|
||||
### Docker
|
||||
|
||||
Simply running `docker compose run --service-ports build --serve` should build the webpages and make them available over localhost.
|
||||
Some more explanation: we are essentially just using docker as a way to provide dependencies and then running the build script. All flags after `build` are passed to `build.sh`. The `service-ports` flag is required to open ports from the container for serving the output, not needed if not using the `--serve` flag of the build script.
|
||||
|
||||
Please note that files generated during the build process using docker are owned by root. This does not cause issues unless you with to manually alter the output or switch to native building instead of docker.
|
||||
|
||||
If you wish to switch to native building after using docker, you must use `sudo git clean -fdx` to remove the files generated using docker.
|
||||
Some more explanation: we are essentially just using docker as a way to provide dependencies and then running the build script. All flags after `build` are passed to `build.py`. The `service-ports` flag is required to open ports from the container for serving the output, not needed if not using the `--serve` flag of the build script.
|
||||
|
||||
## Testing
|
||||
|
||||
While most small changes can be tested adequately by building locally some larger changes, particularly ones relating to the order pages, event registration and other forms may require more integrated testing. This can be achieved using the `test` branch. This branch is built and served in the same way as the main site, [fsfe.org](https://fsfe.org). The built version of the `test` branch may be viewed at [test.fsfe.org](https://test.fsfe.org).
|
||||
|
||||
## Status Viewing
|
||||
|
||||
122
build.py
Executable file
122
build.py
Executable file
@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from build.phase0.full import full
|
||||
from build.phase1.run import phase1_run
|
||||
from build.phase2.run import phase2_run
|
||||
from build.phase3.serve_websites import serve_websites
|
||||
from build.phase3.stage_to_target import stage_to_target
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def parse_arguments() -> argparse.Namespace:
|
||||
"""
|
||||
Parse the arguments of the website build process
|
||||
|
||||
"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Python script to handle building of the fsfe webpage"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
dest="target",
|
||||
help="Final dirs for websites to be build to. Can be a single path, or a comma separated list of valid rsync targets. Supports custom rsynx extension for specifying ports for ssh targets, name@host:path?port.",
|
||||
type=str,
|
||||
default="./output/final",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-level",
|
||||
dest="log_level",
|
||||
type=str,
|
||||
default="INFO",
|
||||
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
||||
help="Set the logging level (default: INFO)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--full",
|
||||
dest="full",
|
||||
help="Force a full rebuild of all webpages.",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--processes",
|
||||
dest="processes",
|
||||
help="Number of processes to use when building the website",
|
||||
type=int,
|
||||
default=multiprocessing.cpu_count(),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--languages",
|
||||
dest="languages",
|
||||
help="Languages to build website in.",
|
||||
default=list(
|
||||
map(lambda path: path.name, Path(".").glob("global/languages/??"))
|
||||
),
|
||||
type=lambda input: input.split(","),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage",
|
||||
dest="stage",
|
||||
help="Force the use of an internal staging directory.",
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--serve",
|
||||
dest="serve",
|
||||
help="Serve the webpages after rebuild",
|
||||
action="store_true",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def main(args: argparse.Namespace):
|
||||
logging.basicConfig(
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
level=args.log_level,
|
||||
)
|
||||
logger.debug(args)
|
||||
|
||||
with multiprocessing.Pool(args.processes) as pool:
|
||||
logger.info("Starting phase 0 - Conditional Setup")
|
||||
|
||||
# TODO Should also be triggered whenever any build python file is changed
|
||||
if args.full:
|
||||
full()
|
||||
|
||||
stage_required = any(
|
||||
[args.stage, "@" in args.target, ":" in args.target, "," in args.target]
|
||||
)
|
||||
working_target = Path("./output/stage" if stage_required else args.target)
|
||||
|
||||
# Processes needed only for subdir stuff
|
||||
phase1_run(args.languages, args.processes, pool)
|
||||
phase2_run(args.languages, pool, working_target)
|
||||
|
||||
logger.info("Starting Phase 3 - Conditional Finishing")
|
||||
if stage_required:
|
||||
stage_to_target(working_target, args.target, pool)
|
||||
|
||||
if args.serve:
|
||||
serve_websites(working_target, 2000, 100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""
|
||||
Main process of the website builder
|
||||
"""
|
||||
# Change to the dir the script is in.
|
||||
os.chdir(os.path.dirname(__file__))
|
||||
args = parse_arguments()
|
||||
main(args)
|
||||
48
build.sh
48
build.sh
@ -1,48 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<-EOF
|
||||
# build.sh Usage
|
||||
## General
|
||||
This script is a wrapper script over ./build/build_main.sh that provides nicer option names, and the options to serve the files.
|
||||
For documentation on the build script itself see ./build/README.md
|
||||
## Flags
|
||||
### -f | --full
|
||||
Perform a full rebuild of the webpages.
|
||||
### -s | --serve
|
||||
Serve the build webpages over localhost.
|
||||
### --
|
||||
Everything after this is passed directly to build_main.
|
||||
See ./build/README.md for valid options.
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
command="build_run"
|
||||
serve=""
|
||||
extra_args=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--full | -f)
|
||||
command="build_into" && shift 1
|
||||
;;
|
||||
--serve | -s)
|
||||
serve="true" && shift 1
|
||||
;;
|
||||
--)
|
||||
shift 1
|
||||
while [ "$#" -gt 0 ]; do
|
||||
extra_args+="$1 "
|
||||
shift 1
|
||||
done
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
mkdir -p ./output
|
||||
./build/build_main.sh "$command" ./output/final --statusdir ./output/final/status.fsfe.org/fsfe.org/data $extra_args
|
||||
if [[ "$serve" ]]; then
|
||||
python3 ./serve-websites.py
|
||||
fi
|
||||
@ -1,42 +0,0 @@
|
||||
# Main Commands
|
||||
Note that targets takes a comma separated list of valid rsync targets, and hence supports ssh targets. If targeting more than one directory one must use the --stage-dir flag documented below.
|
||||
## build_main.sh [options] build_run "targets"
|
||||
Perform the page build. Write output to targets. The source directory is determined from the build scripts own location.
|
||||
|
||||
## build_main.sh [options] git_build_into "targets"
|
||||
Update repo to latest version of upstream branch and then perform a standard build. Write output to targets. The source directory is determined from the build scripts own location.
|
||||
|
||||
## build_main.sh [options] build_into "targets"
|
||||
Perform a full rebuild of the webpages, removing all cached files. Write output to targets. The source directory is determined from the build scripts own location.
|
||||
|
||||
# Internal Commands
|
||||
It is unlikely that you will need to directly call these commands, but they are documented here never the less.
|
||||
## build_main.sh [options] build_xmlstream "file.xhtml"
|
||||
Compile an xml stream from the specified file, additional sources will be determined and included automatically. The stream is suitable for being passed into xsltproc.
|
||||
|
||||
## build_main.sh [options] process_file "file.xhtml" [processor.xsl]
|
||||
Generate output from an xhtml file as if it would be processed during the
|
||||
build. Output is written to STDOUT and can be redirected as desired.
|
||||
If a xslt file is not given, it will be chosen automatically.
|
||||
|
||||
## build_main.sh [options] tree_maker [input_dir] "targets"
|
||||
Generate a set of make rules to build the website contained in input_dir. targets should be the www root of a web server. If input_dir is omitted, it will be the source directory determined from the build scripts location. Note: if targets is set via previous options, and only one parameter is given, then this parameter will be interpreted as input_dir
|
||||
|
||||
# OPTIONS
|
||||
## --source "source_dir"
|
||||
Force a specific source directory. If not explicitly given source_dir is determined from the build scripts own location. Paths given in .sources files are interpreted as relative to source_dir making this option useful when building a webpage outside of the build scripts "regular" tree.
|
||||
|
||||
## --status-dir "status_dir"
|
||||
A directory to which messages are written. If no status_dir is provided information will be written to stdout. The directory will also be used to store some temporary files, which would otherwise be set up in the system wide temp directory.
|
||||
|
||||
## --stage-dir "stage_dir"
|
||||
Directory used for staging the website builds. The websites are first build into this directory, then copied to each targets.
|
||||
|
||||
## --build-env "selection"
|
||||
Indicate the current build environment. "selection" can be one of: * "fsfe.org": building https://fsfe.org on the production server * "test.fsfe.org": building https://test.fsfe.org on the production server * "development" (default): local development build In a local development build, code to dynamically compile the less files into CSS will be included in the HTML output, while in the other environments, the precompiles fsfe.min.css (or valentine.min.css) will be referenced from the generated web pages.
|
||||
|
||||
## --languages "languages"
|
||||
Takes a comma separated list of language shot codes to build the website in. For example, to build the site in English and French only one would use `--languages en,fr`. One of the built languages must be English.
|
||||
|
||||
## --help
|
||||
Show this README.
|
||||
6
build/__init__.py
Normal file
6
build/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
@ -1,94 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
|
||||
|
||||
if [ -z "$inc_arguments" ]; then
|
||||
inc_arguments=true
|
||||
basedir="$(realpath "${0%/*}/..")"
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
-s | --statusdir | --status-dir)
|
||||
[ "$#" -gt 0 ] && shift 1 && statusdir="$1"
|
||||
;;
|
||||
--source)
|
||||
[ "$#" -gt 0 ] && shift 1 && basedir="$1"
|
||||
;;
|
||||
--stage | --stagedir | --stage-dir)
|
||||
[ "$#" -gt 0 ] && shift 1 && stagedir="$1"
|
||||
;;
|
||||
--build-env)
|
||||
[ "$#" -gt 0 ] && shift 1 && build_env="$1"
|
||||
;;
|
||||
--languages)
|
||||
[ "$#" -gt 0 ] && shift 1 && languages="$1"
|
||||
;;
|
||||
-h | --help)
|
||||
command="help"
|
||||
;;
|
||||
build_into)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
git_build_into)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
build_run)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
build_xmlstream)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && workfile="$1"
|
||||
;;
|
||||
tree_maker)
|
||||
command="$1$command"
|
||||
[ -n "$target" -o -n "$3" ] && shift 1 && tree="$1"
|
||||
shift 1
|
||||
[ -n "$1" ] && target="$1"
|
||||
;;
|
||||
process_file)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && workfile="$1"
|
||||
[ "$#" -gt 0 ] && shift 1 && processor="$1"
|
||||
;;
|
||||
*)
|
||||
print_error "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
[ "$#" -gt 0 ] && shift 1
|
||||
done
|
||||
tree="${tree:-$basedir}"
|
||||
stagedir="${stagedir:-$target}"
|
||||
readonly tree="${tree:+$(realpath "$tree")}"
|
||||
readonly stagedir="${stagedir:+$(realpath "$stagedir")}"
|
||||
readonly basedir="${basedir:+$(realpath "$basedir")}"
|
||||
readonly build_env="${build_env:-development}"
|
||||
readonly command
|
||||
if [ "$languages" ]; then
|
||||
readonly languages="$(echo "$languages" | tr ',' ' ')"
|
||||
else
|
||||
readonly languages="$(ls -xw0 "${basedir}/global/languages")"
|
||||
fi
|
||||
if [ "$stagedir" != "$target" ] && printf %s "$target" | egrep -q '^.+@.+:(.+)?$'; then
|
||||
readonly target
|
||||
else
|
||||
readonly target="${target:+$(realpath "$target")}"
|
||||
fi
|
||||
|
||||
case "$command" in
|
||||
build_into) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
git_build_into) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
build_run) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
process_file) [ -z "$workfile" ] && die "Need at least input file" ;;
|
||||
build_xmlstream) [ -z "$workfile" ] && die "Missing xhtml file name" ;;
|
||||
tree_maker) [ -z "$target" ] && die "Missing target location" ;;
|
||||
*help*)
|
||||
cat "$basedir/build/README.md"
|
||||
exit 0
|
||||
;;
|
||||
*) die "Urecognised command or no command given" ;;
|
||||
esac
|
||||
fi
|
||||
@ -1,82 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Dependency check function
|
||||
check_dependencies() {
|
||||
depends="$@"
|
||||
deperrors=''
|
||||
for depend in $depends; do
|
||||
if ! which "$depend" >/dev/null 2>&1; then
|
||||
deperrors="$depend $deperrors"
|
||||
fi
|
||||
done
|
||||
if [ -n "$deperrors" ]; then
|
||||
printf '\033[1;31m'
|
||||
cat <<-EOF
|
||||
The build script depends on some other programs to function.
|
||||
Not all of those programs could be located on your system.
|
||||
Please use your package manager to install the following programs:
|
||||
EOF
|
||||
printf '\n\033[0;31m%s\n' "$deperrors"
|
||||
exit 1
|
||||
fi 1>&2
|
||||
}
|
||||
|
||||
# Check dependencies for all kinds of build envs (e.g. development, fsfe.org)
|
||||
check_dependencies realpath rsync xsltproc xmllint sed find egrep grep wc make tee date iconv wget shuf python3
|
||||
|
||||
if ! make --version | grep -q "GNU Make 4"; then
|
||||
echo "The build script requires GNU Make 4.x"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
basedir="${0%/*}/.."
|
||||
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
|
||||
readonly start_time="$(date +%s)"
|
||||
|
||||
. "$basedir/build/arguments.sh"
|
||||
|
||||
# Check special dependencies for (test.)fsfe.org build server
|
||||
if [ "$build_env" == "fsfe.org" ] || [ "$build_env" == "test.fsfe.org" ]; then
|
||||
check_dependencies lessc
|
||||
fi
|
||||
|
||||
statusdir="${statusdir/#\~/$HOME}"
|
||||
if [ -n "$statusdir" ]; then
|
||||
mkdir -p "$statusdir"
|
||||
[ ! -w "$statusdir" -o ! -d "$statusdir" ] &&
|
||||
die "Unable to set up status directory in \"$statusdir\",\n" \
|
||||
"either select a status directory that exists and is writable,\n" \
|
||||
"or run the build script without output to a status directory"
|
||||
fi
|
||||
readonly statusdir="${statusdir:+$(realpath "$statusdir")}"
|
||||
|
||||
buildpids=$(
|
||||
ps -eo command |
|
||||
egrep "[s]h ${0} .*" |
|
||||
wc -l
|
||||
)
|
||||
if [ $command = "build_into" -o $command = "git_build_into" ] && [ "$buildpids" -gt 2 ]; then
|
||||
debug "build script is already running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh"
|
||||
[ -z "$inc_buildrun" ] && . "$basedir/build/buildrun.sh"
|
||||
[ -z "$inc_makerules" ] && . "$basedir/build/makerules.sh"
|
||||
[ -z "$inc_processor" ] && . "$basedir/build/processor.sh"
|
||||
[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh"
|
||||
|
||||
case "$command" in
|
||||
git_build_into) if [ -f "${statusdir}/full_build" ]; then
|
||||
debug "discovered flag file, performing full build"
|
||||
rm "${statusdir}/full_build"
|
||||
build_into
|
||||
else
|
||||
git_build_into
|
||||
fi ;;
|
||||
build_into) build_into ;;
|
||||
build_run) buildrun ;;
|
||||
process_file) process_file "$workfile" "$processor" ;;
|
||||
build_xmlstream) build_xmlstream "$(get_shortname "$workfile")" "$(get_language "$workfile")" ;;
|
||||
tree_maker) tree_maker "$tree" "$target" ;;
|
||||
esac
|
||||
@ -1,140 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
inc_buildrun=true
|
||||
[ -z "$inc_makerules" ] && . "$basedir/build/makerules.sh"
|
||||
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
|
||||
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
|
||||
|
||||
match() {
|
||||
printf %s "$1" | egrep -q "$2"
|
||||
}
|
||||
|
||||
dir_maker() {
|
||||
# set up directory tree for output
|
||||
# optimise by only issuing mkdir commands
|
||||
# for leaf directories
|
||||
input="${1%/}"
|
||||
output="${2%/}"
|
||||
|
||||
curpath="$output"
|
||||
find "$input" -depth -type d \
|
||||
-regex "$input/[a-z\.]+\.[a-z]+\(/.*\)?" \
|
||||
-printf '%P\n' |
|
||||
while read -r filepath; do
|
||||
oldpath="$curpath"
|
||||
curpath="$output/$filepath/"
|
||||
match "$oldpath" "^$curpath" || mkdir -p "$curpath"
|
||||
done
|
||||
}
|
||||
|
||||
# The actual build
|
||||
buildrun() {
|
||||
set -o pipefail
|
||||
|
||||
printf %s "$start_time" >"$(logname start_time)"
|
||||
|
||||
ncpu="$(grep -c ^processor /proc/cpuinfo)"
|
||||
|
||||
[ -f "$(logname lasterror)" ] && rm "$(logname lasterror)"
|
||||
[ -f "$(logname debug)" ] && rm "$(logname debug)"
|
||||
|
||||
{
|
||||
echo "Starting phase 1" &&
|
||||
make --silent --directory="$basedir" build_env="${build_env}" languages="$languages" 2>&1 &&
|
||||
echo "Finishing phase 1" ||
|
||||
die "Error during phase 1"
|
||||
} | t_logstatus phase_1 || exit 1
|
||||
|
||||
dir_maker "$basedir" "$stagedir" || exit 1
|
||||
|
||||
forcelog Makefile
|
||||
|
||||
{
|
||||
tree_maker "$basedir" "$stagedir" 2>&1 ||
|
||||
die "Error during phase 2 Makefile generation"
|
||||
} >"$(logname Makefile)" || exit 1
|
||||
|
||||
{
|
||||
echo "Starting phase 2" &&
|
||||
make --silent --jobs=$ncpu --file="$(logname Makefile)" 2>&1 &&
|
||||
echo "Finishing phase 2" ||
|
||||
die "Error during phase 2"
|
||||
} | t_logstatus phase_2 || exit 1
|
||||
|
||||
if [ "$stagedir" != "$target" ]; then
|
||||
# rsync issues a "copying unsafe symlink" message for each of the "unsafe"
|
||||
# symlinks which we copy while rsyncing. These messages are issued even if
|
||||
# the files have not changed and clutter up the output, so we filter them
|
||||
# out.
|
||||
{
|
||||
for destination in ${target//,/ }; do
|
||||
echo "Syncing files to $(echo "$destination" | grep -Po "(?<=@)[^:]+")"
|
||||
rsync -av --copy-unsafe-links --del --exclude "status.fsfe.org/*fsfe.org/data" "$stagedir/" "$destination/" | grep -v "copying unsafe symlink"
|
||||
done
|
||||
} | t_logstatus stagesync
|
||||
fi
|
||||
|
||||
date +%s >"$(logname end_time)"
|
||||
|
||||
if [ -n "$statusdir" ]; then
|
||||
(
|
||||
cd "$statusdir"/..
|
||||
./index.cgi | tail -n+3 >"$statusdir"/status_$(date +%s).html
|
||||
)
|
||||
fi
|
||||
}
|
||||
|
||||
# Update git (try 3x) and then do an actual build
|
||||
git_build_into() {
|
||||
forcelog GITchanges
|
||||
GITchanges="$(logname GITchanges)"
|
||||
forcelog GITerrors
|
||||
GITerrors="$(logname GITerrors)"
|
||||
|
||||
gitterm=1
|
||||
i=0
|
||||
while [[ ($gitterm -ne 0) && ($i -lt 3) ]]; do
|
||||
((i++))
|
||||
|
||||
git -C "$basedir" pull >"$GITchanges" 2>"$GITerrors"
|
||||
gitterm="$?"
|
||||
|
||||
if [ $gitterm -ne 0 ]; then
|
||||
debug "Git pull unsuccessful. Trying again in a few seconds."
|
||||
sleep $(shuf -i 10-30 -n1)
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$gitterm" -ne 0 ]; then
|
||||
debug "Three git pulls failed, hard resetting and repulling"
|
||||
git -C "$basedir" reset --hard HEAD~50 >"$GITchanges" 2>"$GITerrors"
|
||||
git -C "$basedir" pull >>"$GITchanges" 2>>"$GITerrors"
|
||||
gitterm="$?"
|
||||
fi
|
||||
|
||||
if [ "$gitterm" -ne 0 ]; then
|
||||
die "GIT reported the following problem:\n$(cat "$GITerrors")"
|
||||
fi
|
||||
|
||||
if egrep '^Already up[ -]to[ -]date' "$GITchanges"; then
|
||||
debug "No changes to GIT:\n$(cat "$GITchanges")"
|
||||
# Exit status should only be 0 if there was a successful build.
|
||||
# So set it to 1 here.
|
||||
exit 1
|
||||
fi
|
||||
|
||||
logstatus GITlatest <"$GITchanges"
|
||||
buildrun
|
||||
}
|
||||
|
||||
# Clean up everything and then do an actual (full) build
|
||||
build_into() {
|
||||
|
||||
# Clean up source directory.
|
||||
git -C "${basedir}" clean -dxf --exclude=status.fsfe.org/translations/data
|
||||
|
||||
# Remove old stage directory.
|
||||
rm -rf "${stagedir}"
|
||||
|
||||
buildrun
|
||||
}
|
||||
@ -1,16 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
inc_filenames=true
|
||||
|
||||
get_language() {
|
||||
# extract language indicator from a given file name
|
||||
echo "$(echo "$1" | sed -r 's:^.*\.([a-z]{2})\.[^\.]+$:\1:')"
|
||||
}
|
||||
|
||||
get_shortname() {
|
||||
# get shortened version of a given file name
|
||||
# required for internal processing
|
||||
|
||||
#echo "$(echo "$1" | sed -r 's:\.[a-z]{2}.xhtml$::')";
|
||||
echo "${1%.??.xhtml}"
|
||||
}
|
||||
6
build/lib/__init__.py
Normal file
6
build/lib/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
128
build/lib/misc.py
Normal file
128
build/lib/misc.py
Normal file
@ -0,0 +1,128 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import lxml.etree as etree
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def keys_exists(element: dict, *keys: str) -> bool:
|
||||
"""
|
||||
Check if *keys (nested) exists in `element` (dict).
|
||||
"""
|
||||
if not isinstance(element, dict):
|
||||
raise AttributeError("keys_exists() expects dict as first argument.")
|
||||
if len(keys) == 0:
|
||||
raise AttributeError("keys_exists() expects at least two arguments, one given.")
|
||||
|
||||
_element = element
|
||||
for key in keys:
|
||||
try:
|
||||
_element = _element[key]
|
||||
except KeyError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def sort_dict(dict: dict) -> dict:
|
||||
"""
|
||||
Sort dict by keys
|
||||
"""
|
||||
return {key: val for key, val in sorted(dict.items(), key=lambda ele: ele[0])}
|
||||
|
||||
|
||||
def update_if_changed(path: Path, content: str) -> None:
|
||||
"""
|
||||
Compare the content of the file at path with the content.
|
||||
If the file does not exist,
|
||||
or its contents does not match content,
|
||||
write content to the file.
|
||||
"""
|
||||
if not path.exists() or path.read_text() != content:
|
||||
logger.debug(f"Updating {path}")
|
||||
path.write_text(content)
|
||||
|
||||
|
||||
def touch_if_newer_dep(file: Path, deps: list[Path]) -> None:
|
||||
"""
|
||||
Takes a filepath , and a list of path of its dependencies.
|
||||
If any of the dependencies has been altered more recently than the file,
|
||||
touch the file.
|
||||
|
||||
Essentially simple reimplementation of make deps for build targets.
|
||||
"""
|
||||
if any(dep.stat().st_mtime > file.stat().st_mtime for dep in deps):
|
||||
logger.info(f"Touching {file}")
|
||||
file.touch()
|
||||
|
||||
|
||||
def delete_file(file: Path) -> None:
|
||||
"""
|
||||
Delete given file using pathlib
|
||||
"""
|
||||
logger.debug(f"Removing file {file}")
|
||||
file.unlink()
|
||||
|
||||
|
||||
def lang_from_filename(file: Path) -> str:
|
||||
"""
|
||||
Get the lang code from a file, where the filename is of format
|
||||
<name>.XX.<ending>, with xx being the lang code.
|
||||
"""
|
||||
lang = file.with_suffix("").suffix.removeprefix(".")
|
||||
# Lang codes should be the iso 631 2 letter codes, but sometimes we use "nolang" to srop a file being built
|
||||
if len(lang) != 2 and lang != "nolang":
|
||||
logger.critical(
|
||||
f"Language {lang} from file {file} not of correct length, exiting"
|
||||
)
|
||||
sys.exit(1)
|
||||
else:
|
||||
return lang
|
||||
|
||||
|
||||
def run_command(commands: list) -> str:
|
||||
result = subprocess.run(
|
||||
commands,
|
||||
capture_output=True,
|
||||
# Get output as str instead of bytes
|
||||
universal_newlines=True,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
logger.critical(f"Command {commands} failed with error")
|
||||
logger.critical(result.stderr.strip())
|
||||
sys.exit(1)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def get_version(file: Path) -> int:
|
||||
"""
|
||||
Get the version tag of an xhtml|xml file
|
||||
"""
|
||||
xslt_tree = etree.parse(Path("build/xslt/get_version.xsl"))
|
||||
transform = etree.XSLT(xslt_tree)
|
||||
result = transform(etree.parse(file))
|
||||
result = str(result).strip()
|
||||
if result == "":
|
||||
result = str(0)
|
||||
logger.debug(f"Got version: {result}")
|
||||
return int(result)
|
||||
|
||||
|
||||
def get_basepath(file: Path) -> Path:
|
||||
"""
|
||||
Return the file with the last two suffixes removed
|
||||
"""
|
||||
return file.with_suffix("").with_suffix("")
|
||||
|
||||
|
||||
def get_basename(file: Path) -> str:
|
||||
"""
|
||||
Return the name of the file with the last two suffixes removed
|
||||
"""
|
||||
return file.with_suffix("").with_suffix("").name
|
||||
248
build/lib/process_file.py
Normal file
248
build/lib/process_file.py
Normal file
@ -0,0 +1,248 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import lxml.etree as etree
|
||||
|
||||
from build.lib.misc import get_basename, get_version, lang_from_filename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _include_xml(file: Path) -> str:
|
||||
"""
|
||||
include second level elements of a given XML file
|
||||
this emulates the behaviour of the original
|
||||
build script which wasn't able to load top
|
||||
level elements from any file
|
||||
"""
|
||||
work_str = ""
|
||||
if file.exists():
|
||||
tree = etree.parse(file)
|
||||
root = tree.getroot()
|
||||
# Remove <version> because the filename attribute would otherwise be added
|
||||
# to this element instead of the actual content element.
|
||||
for elem in root.xpath("version"):
|
||||
root.remove(elem)
|
||||
# Iterate over all elements in root node, add a filename attribute and then append the string to work_str
|
||||
for elem in root.xpath("*"):
|
||||
elem.set("filename", get_basename(file))
|
||||
work_str += etree.tostring(elem, encoding="utf-8").decode("utf-8")
|
||||
|
||||
return work_str
|
||||
|
||||
|
||||
def _get_attributes(file: Path) -> str:
|
||||
"""
|
||||
get attributes of top level element in a given
|
||||
XHTML file
|
||||
"""
|
||||
work_str = ""
|
||||
tree = etree.parse(file)
|
||||
root = tree.getroot()
|
||||
attributes = root.attrib
|
||||
for attrib in attributes:
|
||||
work_str += f'{attrib}="{attributes[attrib]}"\n'
|
||||
|
||||
return work_str
|
||||
|
||||
|
||||
def _list_langs(file: Path) -> str:
|
||||
"""
|
||||
list all languages a file exists in by globbing up
|
||||
the shortname (i.e. file path with file ending omitted)
|
||||
output is readily formatted for inclusion
|
||||
in xml stream
|
||||
"""
|
||||
return "\n".join(
|
||||
list(
|
||||
map(
|
||||
lambda path: (
|
||||
f'<tr id="{lang_from_filename(path)}">'
|
||||
+ (
|
||||
Path(f"global/languages/{lang_from_filename(path)}")
|
||||
.read_text()
|
||||
.strip()
|
||||
if Path(f"global/languages/{lang_from_filename(path)}").exists()
|
||||
else lang_from_filename(path)
|
||||
)
|
||||
+ "</tr>"
|
||||
),
|
||||
file.parent.glob(f"{get_basename(file)}.??{file.suffix}"),
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _auto_sources(action_file: Path, lang: str) -> str:
|
||||
"""
|
||||
import elements from source files, add file name
|
||||
attribute to first element included from each file
|
||||
"""
|
||||
work_str = ""
|
||||
list_file = action_file.with_stem(
|
||||
f".{action_file.with_suffix('').stem}"
|
||||
).with_suffix(".xmllist")
|
||||
|
||||
if list_file.exists():
|
||||
with list_file.open("r") as file:
|
||||
for path in map(lambda line: Path(line.strip()), file):
|
||||
path_xml = (
|
||||
path.with_suffix(f".{lang}.xml")
|
||||
if path.with_suffix(f".{lang}.xml").exists()
|
||||
else path.with_suffix(".en.xml")
|
||||
)
|
||||
work_str += _include_xml(path_xml)
|
||||
|
||||
return work_str
|
||||
|
||||
|
||||
def _build_xmlstream(infile: Path):
|
||||
"""
|
||||
assemble the xml stream for feeding into xsltproc
|
||||
the expected shortname and language flag indicate
|
||||
a single xhtml page to be built
|
||||
"""
|
||||
# TODO
|
||||
# Ideally this would use lxml to construct an object instead of string templating.
|
||||
# Should be a little faster, and also guarantees that its valid xml
|
||||
|
||||
logger.debug(f"infile: {infile}")
|
||||
shortname = infile.with_suffix("")
|
||||
lang = lang_from_filename(infile)
|
||||
glob = infile.parent.joinpath(f"{get_basename(infile)}.??{infile.suffix}")
|
||||
logger.debug(f"formed glob: {glob}")
|
||||
lang_lst = list(
|
||||
infile.parent.glob(f"{get_basename(infile)}.??{infile.suffix}"),
|
||||
)
|
||||
logger.debug(f"file lang list: {lang_lst}")
|
||||
original_lang = (
|
||||
"en"
|
||||
if infile.with_suffix("").with_suffix(f".en{infile.suffix}").exists()
|
||||
else sorted(
|
||||
infile.parent.glob(f"{get_basename(infile)}.??{infile.suffix}"),
|
||||
key=get_version,
|
||||
reverse=True,
|
||||
)[0]
|
||||
.with_suffix("")
|
||||
.suffix.removeprefix(".")
|
||||
)
|
||||
topbanner_xml = Path(f"global/data/topbanner/.topbanner.{lang}.xml")
|
||||
texts_xml = Path(f"global/data/texts/.texts.{lang}.xml")
|
||||
date = str(datetime.now().date())
|
||||
# time = str(datetime.now().time())
|
||||
action_lang = ""
|
||||
translation_state = ""
|
||||
|
||||
if infile.exists():
|
||||
action_lang = lang
|
||||
original_version = get_version(
|
||||
shortname.with_suffix(f".{original_lang}{infile.suffix}")
|
||||
)
|
||||
lang_version = get_version(shortname.with_suffix(f".{lang}{infile.suffix}"))
|
||||
translation_state = (
|
||||
"up-to-date"
|
||||
if (original_version <= lang_version)
|
||||
else (
|
||||
"very-outdated"
|
||||
if (original_version - 3 >= lang_version)
|
||||
else "outdated"
|
||||
)
|
||||
)
|
||||
else:
|
||||
action_lang = original_lang
|
||||
translation_state = "untranslated"
|
||||
|
||||
action_file = shortname.with_suffix(f".{action_lang}{infile.suffix}")
|
||||
logger.debug(f"action_file: {action_file}")
|
||||
|
||||
result_str = f"""
|
||||
<buildinfo
|
||||
date="{date}"
|
||||
original="{original_lang}"
|
||||
filename="/{str(shortname.with_suffix("")).removeprefix("/")}"
|
||||
fileurl="/{shortname.relative_to(shortname.parts[0]).with_suffix("")}"
|
||||
dirname="/{shortname.parent}/"
|
||||
language="{lang}"
|
||||
translation_state="{translation_state}"
|
||||
>
|
||||
|
||||
<trlist>
|
||||
{_list_langs(infile)}
|
||||
</trlist>
|
||||
|
||||
<topbanner>
|
||||
{_include_xml(topbanner_xml)}
|
||||
</topbanner>
|
||||
<textsetbackup>
|
||||
{_include_xml(Path("global/data/texts/texts.en.xml"))}
|
||||
</textsetbackup>
|
||||
<textset>
|
||||
{_include_xml(texts_xml)}
|
||||
</textset>
|
||||
|
||||
<document
|
||||
language="{action_lang}"
|
||||
{_get_attributes(action_file)}
|
||||
>
|
||||
<set>
|
||||
{_auto_sources(action_file, lang)}
|
||||
</set>
|
||||
|
||||
{_include_xml(action_file)}
|
||||
</document>
|
||||
|
||||
</buildinfo>
|
||||
"""
|
||||
return result_str
|
||||
|
||||
|
||||
def process_file(infile: Path, processor: Path) -> str:
|
||||
"""
|
||||
Process a given file using the correct xsl sheet
|
||||
"""
|
||||
logger.debug(f"Processing {infile}")
|
||||
lang = lang_from_filename(infile)
|
||||
xmlstream = _build_xmlstream(infile)
|
||||
xslt_tree = etree.parse(processor.resolve())
|
||||
transform = etree.XSLT(xslt_tree)
|
||||
result = str(transform(etree.XML(xmlstream)))
|
||||
# And now a bunch of regexes to fix some links.
|
||||
# xx is the language code in all comments
|
||||
|
||||
# TODO
|
||||
# Probably a faster way to do this
|
||||
# Maybe iterating though all a tags with lxml?
|
||||
# Once buildxmlstream generates an xml object that should be faster.
|
||||
|
||||
# Remove https://fsfe.org (or https://test.fsfe.org) from the start of all
|
||||
result = re.sub(
|
||||
r"""href\s*=\s*("|')(https?://(test\.)?fsfe\.org)([^>])\1""",
|
||||
r"""href=\1\3\1""",
|
||||
result,
|
||||
flags=re.MULTILINE | re.IGNORECASE,
|
||||
)
|
||||
# Change links from /foo/bar.html into /foo/bar.xx.html
|
||||
# Change links from foo/bar.html into foo/bar.xx.html
|
||||
# Same for .rss and .ics links
|
||||
result = re.sub(
|
||||
r"""href\s*=\s*("|')(/?([^:>]+/)?[^:/.]+\.)(html|rss|ics)(#[^>]*)?\1""",
|
||||
rf"""href=\1\2{lang}.\4\5\1""",
|
||||
result,
|
||||
flags=re.MULTILINE | re.IGNORECASE,
|
||||
)
|
||||
# Change links from /foo/bar/ into /foo/bar/index.xx.html
|
||||
# Change links from foo/bar/ into foo/bar/index.xx.html
|
||||
result = re.sub(
|
||||
r"""href\s*=\s*("|')(/?[^:>]+/)\1""",
|
||||
rf"""href=\1\2index.{lang}.html\1""",
|
||||
result,
|
||||
flags=re.MULTILINE | re.IGNORECASE,
|
||||
)
|
||||
|
||||
return result
|
||||
@ -1,43 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
inc_logging=true
|
||||
|
||||
logname() {
|
||||
name="$1"
|
||||
|
||||
if [ -w "$statusdir" ] && touch "$statusdir/$name"; then
|
||||
echo "$statusdir/$name"
|
||||
elif echo "$forcedlog" | egrep -q "^${name}=.+"; then
|
||||
echo "$forcedlog" |
|
||||
sed -rn "s;^${name}=;;p"
|
||||
else
|
||||
echo /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
forcelog() {
|
||||
name="$1"
|
||||
|
||||
[ "$(logname "$name")" = "/dev/null" ] &&
|
||||
forcedlog="$forcedlog\n${name}=$(mktemp -t w3bldXXXXXXXXX --suffix $$)"
|
||||
}
|
||||
|
||||
[ -z "$USER" ] && USER="$(whoami)"
|
||||
trap "trap - 0 2 3 6 9 15; find \"${TMPDIR:-/tmp}/\" -maxdepth 1 -user \"$USER\" -name \"w3bld*$$\" -delete" 0 2 3 6 9 15
|
||||
|
||||
logstatus() {
|
||||
# pipeline atom to write data streams into a log file
|
||||
tee "$(logname "$1")"
|
||||
}
|
||||
|
||||
t_logstatus() {
|
||||
# pipeline atom to write data streams into a log file
|
||||
while read line; do
|
||||
printf "[$(date +%T)] %s\n" "$line"
|
||||
done | logstatus "$@"
|
||||
}
|
||||
|
||||
logappend() {
|
||||
# pipeline atom to write data streams into a log file
|
||||
tee -a "$(logname "$1")"
|
||||
}
|
||||
@ -1,235 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
inc_makerules=true
|
||||
|
||||
tree_maker() {
|
||||
# walk through file tree and issue Make rules according to file type
|
||||
input="$(realpath "$1")"
|
||||
output="$(realpath "$2")"
|
||||
|
||||
cat <<EOF
|
||||
# -----------------------------------------------------------------------------
|
||||
# Makefile for FSFE website build, phase 2
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
.PHONY: all
|
||||
.DELETE_ON_ERROR:
|
||||
.SECONDEXPANSION:
|
||||
PROCESSOR = "$basedir/build/process_file.sh"
|
||||
PROCFLAGS = --build-env "${build_env}" --source "$basedir"
|
||||
INPUTDIR = $input
|
||||
OUTPUTDIR = $output
|
||||
STATUSDIR = $statusdir
|
||||
LANGUAGES = $languages
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Build .html files from .xhtml sources
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All .xhtml source files
|
||||
HTML_SRC_FILES := \$(shell find "\$(INPUTDIR)" \
|
||||
-name '*.??.xhtml' \
|
||||
-not -path '\$(INPUTDIR)/.git/*' \
|
||||
)
|
||||
|
||||
# All basenames of .xhtml source files (without .<lang>.xhtml ending)
|
||||
# Note: \$(sort ...) is used to remove duplicates
|
||||
HTML_SRC_BASES := \$(sort \$(basename \$(basename \$(HTML_SRC_FILES))))
|
||||
|
||||
# All directories containing .xhtml source files
|
||||
HTML_SRC_DIRS := \$(sort \$(dir \$(HTML_SRC_BASES)))
|
||||
|
||||
# The same as above, but moved to the output directory
|
||||
HTML_DST_BASES := \$(patsubst \$(INPUTDIR)/%,\$(OUTPUTDIR)/%,\$(HTML_SRC_BASES))
|
||||
|
||||
# List of .<lang>.html files to build
|
||||
HTML_DST_FILES := \$(foreach base,\$(HTML_DST_BASES),\$(foreach lang,\$(LANGUAGES),\$(base).\$(lang).html))
|
||||
|
||||
# .xmllist file used to build a html file
|
||||
XMLLIST_DEP = \$(wildcard \$(INPUTDIR)/\$(dir \$*).\$(notdir \$*).xmllist)
|
||||
|
||||
# .xsl file used to build a html file
|
||||
XSL_DEP = \$(firstword \$(wildcard \$(INPUTDIR)/\$*.xsl) \$(INPUTDIR)/\$(dir \$*).default.xsl)
|
||||
|
||||
all: \$(HTML_DST_FILES)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(filter %.${lang}.html,\$(HTML_DST_FILES)): \$(OUTPUTDIR)/%.${lang}.html: \$(INPUTDIR)/%.*.xhtml \$\$(XMLLIST_DEP) \$\$(XSL_DEP) \$(INPUTDIR)/global/data/texts/.texts.${lang}.xml \$(INPUTDIR)/global/data/texts/texts.en.xml \$(INPUTDIR)/global/data/topbanner/.topbanner.${lang}.xml
|
||||
echo "* Building \$*.${lang}.html"
|
||||
\${PROCESSOR} \${PROCFLAGS} process_file "\$(INPUTDIR)/\$*.${lang}.xhtml" > "\$@"
|
||||
EOF
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create index.* symlinks
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All .xhtml source files with the same name as their parent directory
|
||||
INDEX_SRC_FILES := \$(wildcard \$(foreach directory,\$(HTML_SRC_DIRS),\$(directory)\$(notdir \$(directory:/=)).??.xhtml))
|
||||
|
||||
# All basenames of .xhtml source files with the same name as their parent
|
||||
# directory
|
||||
INDEX_SRC_BASES := \$(sort \$(basename \$(basename \$(INDEX_SRC_FILES))))
|
||||
|
||||
# All directories containing .xhtml source files with the same name as their
|
||||
# parent directory (that is, all directories in which index files should be
|
||||
# created)
|
||||
INDEX_SRC_DIRS := \$(dir \$(INDEX_SRC_BASES))
|
||||
|
||||
# The same as above, but moved to the output directory
|
||||
INDEX_DST_DIRS := \$(patsubst \$(INPUTDIR)/%,\$(OUTPUTDIR)/%,\$(INDEX_SRC_DIRS))
|
||||
|
||||
# List of index.<lang>.html symlinks to create
|
||||
INDEX_DST_LINKS := \$(foreach base,\$(INDEX_DST_DIRS),\$(foreach lang,\$(LANGUAGES),\$(base)index.\$(lang).html))
|
||||
|
||||
all: \$(INDEX_DST_LINKS)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(filter %/index.${lang}.html,\$(INDEX_DST_LINKS)): \$(OUTPUTDIR)/%/index.${lang}.html:
|
||||
echo "* Creating symlink \$*/index.${lang}.html"
|
||||
ln -sf "\$(notdir \$*).${lang}.html" "\$@"
|
||||
EOF
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create symlinks from file.<lang>.html to file.html.<lang>
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# List of .html.<lang> symlinks to create
|
||||
HTML_DST_LINKS := \$(foreach base,\$(HTML_DST_BASES) \$(addsuffix index,\$(INDEX_DST_DIRS)),\$(foreach lang,\$(LANGUAGES),\$(base).html.\$(lang)))
|
||||
|
||||
all: \$(HTML_DST_LINKS)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(OUTPUTDIR)/%.html.${lang}:
|
||||
echo "* Creating symlink \$*.html.${lang}"
|
||||
ln -sf "\$(notdir \$*).${lang}.html" "\$@"
|
||||
EOF
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Build .rss files from .xhtml sources
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All .rss.xsl scripts which can create .rss output
|
||||
RSS_SRC_SCRIPTS := \$(shell find "\$(INPUTDIR)" \
|
||||
-name '*.rss.xsl' \
|
||||
-not -path '\$(INPUTDIR)/.git/*' \
|
||||
)
|
||||
|
||||
# All basenames of .xhtml source files from which .rss files should be built
|
||||
RSS_SRC_BASES := \$(sort \$(basename \$(basename \$(RSS_SRC_SCRIPTS))))
|
||||
|
||||
# The same as above, but moved to the output directory
|
||||
RSS_DST_BASES := \$(patsubst \$(INPUTDIR)/%,\$(OUTPUTDIR)/%,\$(RSS_SRC_BASES))
|
||||
|
||||
# List of .<lang>.rss files to build
|
||||
RSS_DST_FILES := \$(foreach base,\$(RSS_DST_BASES),\$(foreach lang,\$(LANGUAGES),\$(base).\$(lang).rss))
|
||||
|
||||
all: \$(RSS_DST_FILES)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(OUTPUTDIR)/%.${lang}.rss: \$(INPUTDIR)/%.*.xhtml \$\$(XMLLIST_DEP) \$(INPUTDIR)/%.rss.xsl \$(INPUTDIR)/global/data/texts/.texts.${lang}.xml \$(INPUTDIR)/global/data/texts/texts.en.xml
|
||||
echo "* Building \$*.${lang}.rss"
|
||||
\${PROCESSOR} \${PROCFLAGS} process_file "\$(INPUTDIR)/\$*.${lang}.xhtml" "\$(INPUTDIR)/\$*.rss.xsl" > "\$@"
|
||||
EOF
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Build .ics files from .xhtml sources
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All .ics.xsl scripts which can create .ics output
|
||||
ICS_SRC_SCRIPTS := \$(shell find "\$(INPUTDIR)" \
|
||||
-name '*.ics.xsl' \
|
||||
-not -path '\$(INPUTDIR)/.git/*' \
|
||||
)
|
||||
|
||||
# All basenames of .xhtml source files from which .ics files should be built
|
||||
ICS_SRC_BASES := \$(sort \$(basename \$(basename \$(ICS_SRC_SCRIPTS))))
|
||||
|
||||
# The same as above, but moved to the output directory
|
||||
ICS_DST_BASES := \$(patsubst \$(INPUTDIR)/%,\$(OUTPUTDIR)/%,\$(ICS_SRC_BASES))
|
||||
|
||||
# List of .<lang>.ics files to build
|
||||
ICS_DST_FILES := \$(foreach base,\$(ICS_DST_BASES),\$(foreach lang,\$(LANGUAGES),\$(base).\$(lang).ics))
|
||||
|
||||
all: \$(ICS_DST_FILES)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(OUTPUTDIR)/%.${lang}.ics: \$(INPUTDIR)/%.*.xhtml \$\$(XMLLIST_DEP) \$(INPUTDIR)/%.ics.xsl \$(INPUTDIR)/global/data/texts/.texts.${lang}.xml \$(INPUTDIR)/global/data/texts/texts.en.xml
|
||||
echo "* Building \$*.${lang}.ics"
|
||||
\${PROCESSOR} \${PROCFLAGS} process_file "\$(INPUTDIR)/\$*.${lang}.xhtml" "\$(INPUTDIR)/\$*.ics.xsl" > "\$@"
|
||||
EOF
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Copy images, docments etc
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All files which should just be copied over
|
||||
COPY_SRC_FILES := \$(shell find -L "\$(INPUTDIR)" -type f \
|
||||
-regex "\$(INPUTDIR)/[a-z\.]+\.[a-z]+/.*" \
|
||||
-not -name '.drone.yml' \
|
||||
-not -name '.gitignore' \
|
||||
-not -name 'README*' \
|
||||
-not -name 'Makefile' \
|
||||
-not -name '*.sources' \
|
||||
-not -name "*.xmllist" \
|
||||
-not -name '*.xhtml' \
|
||||
-not -name '*.xml' \
|
||||
-not -name '*.xsl' \
|
||||
-not -name '*.nix' \
|
||||
) \$(INPUTDIR)/fsfe.org/order/data/items.en.xml
|
||||
|
||||
# The same as above, but moved to the output directory
|
||||
COPY_DST_FILES := \$(sort \$(patsubst \$(INPUTDIR)/%,\$(OUTPUTDIR)/%,\$(COPY_SRC_FILES)))
|
||||
|
||||
all: \$(COPY_DST_FILES)
|
||||
\$(COPY_DST_FILES): \$(OUTPUTDIR)/%: \$(INPUTDIR)/%
|
||||
echo "* Copying file \$*"
|
||||
rsync -l "\$<" "\$@"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Clean up excess files in target directory
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
ALL_DST := \$(HTML_DST_FILES) \$(INDEX_DST_LINKS) \$(HTML_DST_LINKS) \$(RSS_DST_FILES) \$(ICS_DST_FILES) \$(COPY_DST_FILES) \$(SOURCE_DST_FILES)
|
||||
|
||||
.PHONY: clean
|
||||
all: clean
|
||||
clean:
|
||||
# Write all destination filenames into "manifest" file, one per line
|
||||
\$(file >\$(STATUSDIR)/manifest)
|
||||
\$(foreach filename,\$(ALL_DST),\$(file >>\$(STATUSDIR)/manifest,\$(filename)))
|
||||
sort "\$(STATUSDIR)/manifest" > "\$(STATUSDIR)/manifest.sorted"
|
||||
find -L "\$(OUTPUTDIR)" -type f -path "\$(STATUSDIR)" -prune \\
|
||||
| sort \\
|
||||
| diff - "\$(STATUSDIR)/manifest.sorted" \\
|
||||
| sed -rn 's;^< ;;p' \\
|
||||
| while read file; do echo "* Deleting \$\${file}"; rm "\$\${file}"; done
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
EOF
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
inc_misc=true
|
||||
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
|
||||
|
||||
debug() {
|
||||
if [ "$#" -ge 1 ]; then
|
||||
echo "$(date '+%F %T'): $@" | logappend debug >&2
|
||||
else
|
||||
logappend debug >&2
|
||||
fi
|
||||
}
|
||||
|
||||
print_error() {
|
||||
echo "Error - $@" | logappend lasterror >&2
|
||||
echo "Run '$0 --help' to see usage instructions" >&2
|
||||
}
|
||||
|
||||
die() {
|
||||
echo "$(date '+%F %T'): Fatal - $@" | logappend lasterror >&2
|
||||
date +%s | logstatus end_time
|
||||
exit 1
|
||||
}
|
||||
20
build/phase0/full.py
Normal file
20
build/phase0/full.py
Normal file
@ -0,0 +1,20 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
|
||||
from build.lib.misc import run_command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def full() -> None:
|
||||
"""
|
||||
Git clean the repo to remove all cached artifacts
|
||||
Excluded the root .venv repo, as removing it mid build breaks the build, obviously
|
||||
"""
|
||||
logger.info("Performing a full rebuild, git cleaning")
|
||||
run_command(
|
||||
["git", "clean", "-fdx", "--exclude", "/.venv"],
|
||||
)
|
||||
6
build/phase1/__init__.py
Normal file
6
build/phase1/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
62
build/phase1/create_activities_file.py
Normal file
62
build/phase1/create_activities_file.py
Normal file
@ -0,0 +1,62 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import csv
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
raw_url = urlparse("https://git.fsfe.org/FSFE/activities/raw/branch/master/activities.csv")
|
||||
|
||||
def create_activities_file():
|
||||
git_token = os.environ.get("GIT_TOKEN")
|
||||
if git_token is None:
|
||||
logger.warn("GIT_TOKEN is not set, skipping generation of activities file")
|
||||
return
|
||||
|
||||
url = raw_url._replace(query=f"token={git_token}").geturl()
|
||||
r = requests.get(url)
|
||||
|
||||
if not r.ok:
|
||||
logger.error("Failed to retrieve activities file")
|
||||
raise Exception("Failed to retrieve activities file")
|
||||
|
||||
activities_csv = csv.reader(r.text.split("\n")[1:], delimiter="\t")
|
||||
activities = ""
|
||||
|
||||
for row in activities_csv:
|
||||
if len(row) == 0:
|
||||
continue
|
||||
|
||||
tag = row[0]
|
||||
description = row[1]
|
||||
event = row[2]
|
||||
|
||||
activities += f' <option value="{tag}||{description}"'
|
||||
if event:
|
||||
activities += f' data-event="{event}"'
|
||||
|
||||
activities += '>'
|
||||
activities += f"{tag} ({description})"
|
||||
activities += "</option>\n"
|
||||
|
||||
content = f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<data>
|
||||
<version>1</version>
|
||||
|
||||
<module>
|
||||
{activities}
|
||||
</module>
|
||||
</data>"""
|
||||
|
||||
activities_path = Path("global/data/modules/fsfe-activities-options.en.xml")
|
||||
with open(activities_path, "w") as f:
|
||||
f.write(content)
|
||||
|
||||
logger.info(f"Wrote activity file to {str(activities_path)}")
|
||||
39
build/phase1/global_symlinks.py
Normal file
39
build/phase1/global_symlinks.py
Normal file
@ -0,0 +1,39 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from itertools import product
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _do_symlinking(type: str, lang: str) -> None:
|
||||
"""
|
||||
Helper function for doing all of the global symlinking that is suitable for multithreading
|
||||
"""
|
||||
target = (
|
||||
Path(f"global/data/{type}/{type}.{lang}.xml")
|
||||
if Path(f"global/data/{type}/{type}.{lang}.xml").exists()
|
||||
else Path(f"global/data/{type}/{type}.en.xml")
|
||||
)
|
||||
source = Path(f"global/data/{type}/.{type}.{lang}.xml")
|
||||
if not source.exists():
|
||||
source.symlink_to(target.relative_to(source.parent))
|
||||
|
||||
|
||||
def global_symlinks(languages: list[str], pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
After this step, the following symlinks will exist:
|
||||
* global/data/texts/.texts.<lang>.xml for each language
|
||||
* global/data/topbanner/.topbanner.<lang>.xml for each language
|
||||
Each of these symlinks will point to the corresponding file without a dot at
|
||||
the beginning of the filename, if present, and to the English version
|
||||
otherwise. This symlinks make sure that phase 2 can easily use the right file
|
||||
for each language, also as a prerequisite in the Makefile.
|
||||
"""
|
||||
logger.info("Creating global symlinks")
|
||||
types = ["texts", "topbanner"]
|
||||
pool.starmap(_do_symlinking, product(types, languages))
|
||||
121
build/phase1/index_website.py
Normal file
121
build/phase1/index_website.py
Normal file
@ -0,0 +1,121 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# Build an index for the search engine based on the article titles and tags
|
||||
|
||||
import json
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
import iso639
|
||||
import lxml.etree as etree
|
||||
import nltk
|
||||
from nltk.corpus import stopwords as nltk_stopwords
|
||||
|
||||
from build.lib.misc import update_if_changed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _find_teaser(document: etree.ElementTree) -> str:
|
||||
"""
|
||||
Find a suitable teaser for indexation
|
||||
|
||||
Get all the paragraphs in <body> and return the first which contains more
|
||||
than 10 words
|
||||
|
||||
:document: The parsed lxml ElementTree document
|
||||
:returns: The text of the teaser or an empty string
|
||||
"""
|
||||
for p in document.xpath("//body//p"):
|
||||
if p.text and len(p.text.strip().split(" ")) > 10:
|
||||
return p.text
|
||||
return ""
|
||||
|
||||
|
||||
def _process_file(file: Path, stopwords: set[str]) -> dict:
|
||||
"""
|
||||
Generate the search index entry for a given file and set of stopwords
|
||||
"""
|
||||
logger.debug(f"Processing file {file}")
|
||||
xslt_root = etree.parse(file)
|
||||
tags = map(
|
||||
lambda tag: tag.get("key"),
|
||||
filter(lambda tag: tag.get("key") != "front-page", xslt_root.xpath("//tag")),
|
||||
)
|
||||
return {
|
||||
"url": f"/{file.with_suffix('.html')}",
|
||||
"tags": " ".join(tags),
|
||||
"title": (
|
||||
xslt_root.xpath("//html//title")[0].text
|
||||
if xslt_root.xpath("//html//title")
|
||||
else ""
|
||||
),
|
||||
"teaser": " ".join(
|
||||
w
|
||||
for w in _find_teaser(xslt_root).strip().split(" ")
|
||||
if w.lower() not in stopwords
|
||||
),
|
||||
"type": "news" if "news/" in str(file) else "page",
|
||||
# Get the date of the file if it has one
|
||||
"date": (
|
||||
xslt_root.xpath("//news[@newsdate]").get("newsdate")
|
||||
if xslt_root.xpath("//news[@newsdate]")
|
||||
else None
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def index_websites(languages: list[str], pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
Generate a search index for all sites that have a search/search.js file
|
||||
"""
|
||||
logger.info("Creating search indexes")
|
||||
# Download all stopwords
|
||||
nltkdir = "./.nltk_data"
|
||||
nltk.data.path = [nltkdir] + nltk.data.path
|
||||
nltk.download("stopwords", download_dir=nltkdir, quiet=True)
|
||||
# Iterate over sites
|
||||
for site in filter(
|
||||
lambda path: path.joinpath("search/search.js").exists(),
|
||||
Path(".").glob("?*.??*"),
|
||||
):
|
||||
logger.debug(f"Indexing {site}")
|
||||
|
||||
# Get all xhtml files in languages to be processed
|
||||
# Create a list of tuples
|
||||
# The first element of each tuple is the file and the second is a set of stopwords for that language
|
||||
# Use iso639 to get the english name of the language from the two letter iso639-1 code we use to mark files.
|
||||
# Then if that language has stopwords from nltk, use those stopwords.
|
||||
files_with_stopwords = map(
|
||||
lambda file: (
|
||||
file,
|
||||
(
|
||||
set(
|
||||
nltk_stopwords.words(
|
||||
iso639.Language.from_part1(
|
||||
file.suffixes[0].removeprefix(".")
|
||||
).name.lower()
|
||||
)
|
||||
)
|
||||
if iso639.Language.from_part1(
|
||||
file.suffixes[0].removeprefix(".")
|
||||
).name.lower()
|
||||
in nltk_stopwords.fileids()
|
||||
else set()
|
||||
),
|
||||
),
|
||||
filter(
|
||||
lambda file: file.suffixes[0].removeprefix(".") in languages,
|
||||
Path(site).glob("**/*.??.xhtml"),
|
||||
),
|
||||
)
|
||||
|
||||
articles = pool.starmap(_process_file, files_with_stopwords)
|
||||
|
||||
update_if_changed(
|
||||
Path(f"{site}/search/index.js"),
|
||||
"var pages = " + json.dumps(articles, ensure_ascii=False),
|
||||
)
|
||||
30
build/phase1/prepare_subdirectories.py
Normal file
30
build/phase1/prepare_subdirectories.py
Normal file
@ -0,0 +1,30 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def prepare_subdirectories(languages: list[str], processes: int) -> None:
|
||||
"""
|
||||
Find any makefiles in subdirectories and run them
|
||||
"""
|
||||
logger.info("Preparing Subdirectories")
|
||||
for subdir_path in map(
|
||||
lambda path: path.parent, Path("").glob("?*.?*/**/subdir.py")
|
||||
):
|
||||
logger.info(f"Preparing subdirectory {subdir_path}")
|
||||
sys.path.append(str(subdir_path.resolve()))
|
||||
import subdir
|
||||
|
||||
subdir.run(languages, processes, subdir_path)
|
||||
# Remove its path from where things can be imported
|
||||
sys.path.remove(str(subdir_path.resolve()))
|
||||
# Remove it from loaded modules
|
||||
sys.modules.pop("subdir")
|
||||
# prevent us from accessing it again
|
||||
del subdir
|
||||
128
build/phase1/run.py
Normal file
128
build/phase1/run.py
Normal file
@ -0,0 +1,128 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# script for FSFE website build, phase 1
|
||||
# -----------------------------------------------------------------------------
|
||||
# This script is executed in the root of the source directory tree, and
|
||||
# creates some .xml and xhtml files as well as some symlinks, all of which
|
||||
# serve as input files in phase 2. The whole phase 1 runs within the source
|
||||
# directory tree and does not touch the target directory tree at all.
|
||||
# -----------------------------------------------------------------------------
|
||||
import logging
|
||||
import multiprocessing
|
||||
|
||||
from .create_activities_file import create_activities_file
|
||||
from .global_symlinks import global_symlinks
|
||||
from .index_website import index_websites
|
||||
from .prepare_subdirectories import prepare_subdirectories
|
||||
from .update_css import update_css
|
||||
from .update_defaultxsls import update_defaultxsls
|
||||
from .update_localmenus import update_localmenus
|
||||
from .update_stylesheets import update_stylesheets
|
||||
from .update_tags import update_tags
|
||||
from .update_xmllists import update_xmllists
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def phase1_run(languages: list[str], processes: int, pool: multiprocessing.Pool):
|
||||
"""
|
||||
Run all the necessary sub functions for phase1.
|
||||
"""
|
||||
logger.info("Starting Phase 1 - Setup")
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create XML activities file
|
||||
# -----------------------------------------------------------------------------
|
||||
create_activities_file()
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Build search index
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# This step runs a Python tool that creates an index of all news and
|
||||
# articles. It extracts titles, teaser, tags, dates and potentially more.
|
||||
# The result will be fed into a JS file.
|
||||
index_websites(languages, pool)
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update CSS files
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# This step recompiles the less files into the final CSS files to be
|
||||
# distributed to the web server.
|
||||
update_css()
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update XSL stylesheets
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# This step updates (actually: just touches) all XSL files which depend on
|
||||
# another XSL file that has changed since the last build run. The phase 2
|
||||
# Makefile then only has to consider the directly used stylesheet as a
|
||||
# prerequisite for building each file and doesn't have to worry about other
|
||||
# stylesheets imported into that one.
|
||||
# This must run before the "dive into subdirectories" step, because in the news
|
||||
# and events directories, the XSL files, if updated, will be copied for the
|
||||
# per-year archives.
|
||||
|
||||
update_stylesheets(pool)
|
||||
# -----------------------------------------------------------------------------
|
||||
# Dive into subdirectories
|
||||
# -----------------------------------------------------------------------------
|
||||
# Find any makefiles in subdirectories and run them
|
||||
prepare_subdirectories(languages, processes)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create XML symlinks
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, the following symlinks will exist:
|
||||
# * global/data/texts/.texts.<lang>.xml for each language
|
||||
# * global/data/topbanner/.topbanner.<lang>.xml for each language
|
||||
# Each of these symlinks will point to the corresponding file without a dot at
|
||||
# the beginning of the filename, if present, and to the English version
|
||||
# otherwise. This symlinks make sure that phase 2 can easily use the right file
|
||||
# for each language, also as a prerequisite in the Makefile.
|
||||
global_symlinks(languages, pool)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create XSL symlinks
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, each directory with source files for HTML pages contains a
|
||||
# symlink named .default.xsl and pointing to the default.xsl "responsible" for
|
||||
# this directory. These symlinks make it easier for the phase 2 Makefile to
|
||||
# determine which XSL script should be used to build a HTML page from a source
|
||||
# file.
|
||||
|
||||
update_defaultxsls(pool)
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update local menus
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, all .localmenu.??.xml files will be up to date.
|
||||
|
||||
update_localmenus(languages, pool)
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update tags
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, the following files will be up to date:
|
||||
# * tags/tagged-<tags>.en.xhtml for each tag used. Apart from being
|
||||
# automatically created, these are regular source files for HTML pages, and
|
||||
# in phase 2 are built into pages listing all news items and events for a
|
||||
# tag.
|
||||
# * tags/.tags.??.xml with a list of the tags used.
|
||||
update_tags(languages, pool)
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update XML filelists
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# After this step, the following files will be up to date:
|
||||
# * <dir>/.<base>.xmllist for each <dir>/<base>.sources as well as for each
|
||||
# $site/tags/tagged-<tags>.en.xhtml. These files are used in phase 2 to include the
|
||||
# correct XML files when generating the HTML pages. It is taken care that
|
||||
# these files are only updated whenever their content actually changes, so
|
||||
# they can serve as a prerequisite in the phase 2 Makefile.
|
||||
update_xmllists(languages, pool)
|
||||
44
build/phase1/update_css.py
Normal file
44
build/phase1/update_css.py
Normal file
@ -0,0 +1,44 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import minify
|
||||
|
||||
from build.lib.misc import run_command, update_if_changed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def update_css() -> None:
|
||||
"""
|
||||
If any less files have been changed, update the css.
|
||||
Compile less found at website/look/(fsfe.less|valentine.less)
|
||||
Then minify it, and place it in the expected location for the build process.
|
||||
"""
|
||||
logger.info("Updating css")
|
||||
for dir in Path("").glob("?*.?*/look"):
|
||||
for name in ["fsfe", "valentine"]:
|
||||
if dir.joinpath(name + ".less").exists() and (
|
||||
not dir.joinpath(name + ".min.css").exists()
|
||||
or any(
|
||||
map(
|
||||
lambda path: path.stat().st_mtime
|
||||
> dir.joinpath(name + ".min.css").stat().st_mtime,
|
||||
dir.glob("**/*.less"),
|
||||
)
|
||||
)
|
||||
):
|
||||
logger.info(f"Compiling {name}.less")
|
||||
result = run_command(
|
||||
[
|
||||
"lessc",
|
||||
str(dir.joinpath(name + ".less")),
|
||||
],
|
||||
)
|
||||
update_if_changed(
|
||||
dir.joinpath(name + ".min.css"),
|
||||
minify.string("text/css", result),
|
||||
)
|
||||
41
build/phase1/update_defaultxsls.py
Executable file
41
build/phase1/update_defaultxsls.py
Executable file
@ -0,0 +1,41 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _do_symlinking(directory: Path) -> None:
|
||||
"""
|
||||
In each dir, place a .default.xsl symlink pointing to the nearest default.xsl
|
||||
"""
|
||||
working_dir = directory
|
||||
if not directory.joinpath(".default.xsl").exists():
|
||||
while not working_dir.joinpath("default.xsl").exists():
|
||||
working_dir = working_dir.parent
|
||||
directory.joinpath(".default.xsl").symlink_to(
|
||||
working_dir.joinpath("default.xsl").resolve()
|
||||
)
|
||||
|
||||
|
||||
def update_defaultxsls(pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
Place a .default.xsl into each directory containing source files for
|
||||
HTML pages (*.xhtml). These .default.xsl are symlinks to the first
|
||||
available actual default.xsl found when climbing the directory tree
|
||||
upwards, it's the xsl stylesheet to be used for building the HTML
|
||||
files from this directory.
|
||||
"""
|
||||
logger.info("Updating default xsl's")
|
||||
|
||||
# Get a set of all directories containing .xhtml source files
|
||||
directories = set(
|
||||
map(lambda path: path.parent, Path(".").glob("*?.?*/**/*.*.xhtml"))
|
||||
)
|
||||
|
||||
# Do all directories asynchronously
|
||||
pool.map(_do_symlinking, directories)
|
||||
118
build/phase1/update_localmenus.py
Executable file
118
build/phase1/update_localmenus.py
Executable file
@ -0,0 +1,118 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
import lxml.etree as etree
|
||||
|
||||
from build.lib.misc import get_basepath
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _write_localmenus(
|
||||
dir: str, files_by_dir: dict[str, list[Path]], languages: list[str]
|
||||
) -> None:
|
||||
"""
|
||||
Write localmenus for a given directory
|
||||
"""
|
||||
# Set of files with no langcode or xhtml extension
|
||||
base_files = set(
|
||||
map(
|
||||
lambda filter_file: get_basepath(filter_file),
|
||||
files_by_dir[dir],
|
||||
)
|
||||
)
|
||||
for lang in languages:
|
||||
file = Path(dir).joinpath(f".localmenu.{lang}.xml")
|
||||
logger.debug(f"Creating {file}")
|
||||
page = etree.Element("feed")
|
||||
|
||||
# Add the subelements
|
||||
version = etree.SubElement(page, "version")
|
||||
version.text = "1"
|
||||
|
||||
for source_file in filter(
|
||||
lambda path: path is not None,
|
||||
map(
|
||||
lambda base_file: base_file.with_suffix(f".{lang}.xhtml")
|
||||
if base_file.with_suffix(f".{lang}.xhtml").exists()
|
||||
else (
|
||||
base_file.with_suffix(".en.xhtml")
|
||||
if base_file.with_suffix(".en.xhtml").exists()
|
||||
else None
|
||||
),
|
||||
base_files,
|
||||
),
|
||||
):
|
||||
for localmenu in etree.parse(source_file).xpath("//localmenu"):
|
||||
etree.SubElement(
|
||||
page,
|
||||
"localmenuitem",
|
||||
set=(
|
||||
str(localmenu.xpath("./@set")[0])
|
||||
if localmenu.xpath("./@set") != []
|
||||
else "default"
|
||||
),
|
||||
id=(
|
||||
str(localmenu.xpath("./@id")[0])
|
||||
if localmenu.xpath("./@id") != []
|
||||
else "default"
|
||||
),
|
||||
link=(
|
||||
str(
|
||||
source_file.with_suffix(".html").relative_to(
|
||||
source_file.parents[0]
|
||||
)
|
||||
)
|
||||
),
|
||||
).text = localmenu.text
|
||||
|
||||
page.getroottree().write(file, xml_declaration=True, encoding="utf-8")
|
||||
|
||||
|
||||
def update_localmenus(languages: list[str], pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
Update all the .localmenu.*.xml files containing the local menus.
|
||||
"""
|
||||
logger.info("Updating local menus")
|
||||
# Get a dict of all source files containing local menus
|
||||
files_by_dir = {}
|
||||
for file in filter(
|
||||
lambda path: "-template" not in str(path),
|
||||
Path(".").glob("*?.?*/**/*.??.xhtml"),
|
||||
):
|
||||
xslt_root = etree.parse(file)
|
||||
if xslt_root.xpath("//localmenu"):
|
||||
dir = xslt_root.xpath("//localmenu/@dir")
|
||||
dir = dir[0] if dir else str(file.parent.relative_to(Path(".")))
|
||||
if dir not in files_by_dir:
|
||||
files_by_dir[dir] = set()
|
||||
files_by_dir[dir].add(file)
|
||||
for dir in files_by_dir:
|
||||
files_by_dir[dir] = sorted(list(files_by_dir[dir]))
|
||||
|
||||
# If any of the source files has been updated, rebuild all .localmenu.*.xml
|
||||
dirs = filter(
|
||||
lambda dir: (
|
||||
any(
|
||||
map(
|
||||
lambda file: (
|
||||
(not Path(dir).joinpath(".localmenu.en.xml").exists())
|
||||
or (
|
||||
file.stat().st_mtime
|
||||
> Path(dir).joinpath(".localmenu.en.xml").stat().st_mtime
|
||||
)
|
||||
),
|
||||
files_by_dir[dir],
|
||||
)
|
||||
)
|
||||
),
|
||||
files_by_dir,
|
||||
)
|
||||
pool.starmap(
|
||||
_write_localmenus, map(lambda dir: (dir, files_by_dir, languages), dirs)
|
||||
)
|
||||
49
build/phase1/update_stylesheets.py
Executable file
49
build/phase1/update_stylesheets.py
Executable file
@ -0,0 +1,49 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from lxml import etree
|
||||
|
||||
from build.lib.misc import touch_if_newer_dep
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _update_sheet(file: Path) -> None:
|
||||
"""
|
||||
Update a given xsl file if any of its dependant xsl files have been updated
|
||||
"""
|
||||
xslt_root = etree.parse(file)
|
||||
imports = map(
|
||||
lambda imp: file.parent.joinpath(imp.get("href"))
|
||||
.resolve()
|
||||
.relative_to(Path(".").resolve()),
|
||||
xslt_root.xpath(
|
||||
"//xsl:import", namespaces={"xsl": "http://www.w3.org/1999/XSL/Transform"}
|
||||
),
|
||||
)
|
||||
touch_if_newer_dep(file, imports)
|
||||
|
||||
|
||||
def update_stylesheets(pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
This script is called from the phase 1 Makefile and touches all XSL files
|
||||
which depend on another XSL file that has changed since the last build run.
|
||||
The phase 2 Makefile then only has to consider the
|
||||
directly used stylesheet as a prerequisite for building each file and doesn't
|
||||
have to worry about other stylesheets imported into that one.
|
||||
"""
|
||||
logger.info("Updating XSL stylesheets")
|
||||
banned = re.compile(r"(\.venv/.*)|(.*\.default\.xsl$)")
|
||||
pool.map(
|
||||
_update_sheet,
|
||||
filter(
|
||||
lambda file: re.match(banned, str(file)) is None,
|
||||
Path(".").glob("**/*.xsl"),
|
||||
),
|
||||
)
|
||||
179
build/phase1/update_tags.py
Executable file
179
build/phase1/update_tags.py
Executable file
@ -0,0 +1,179 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
from xml.sax.saxutils import escape
|
||||
|
||||
import lxml.etree as etree
|
||||
|
||||
from build.lib.misc import (
|
||||
get_basepath,
|
||||
keys_exists,
|
||||
lang_from_filename,
|
||||
sort_dict,
|
||||
update_if_changed,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _update_tag_pages(site: Path, tag: str, languages: list[str]) -> None:
|
||||
"""
|
||||
Update the xhtml pages and xmllists for a given tag
|
||||
"""
|
||||
for lang in languages:
|
||||
tagfile_source = Path(f"{site}/tags/tagged.{lang}.xhtml")
|
||||
if tagfile_source.exists():
|
||||
taggedfile = Path(f"{site}/tags/tagged-{tag}.{lang}.xhtml")
|
||||
content = tagfile_source.read_text().replace("XXX_TAGNAME_XXX", tag)
|
||||
update_if_changed(taggedfile, content)
|
||||
|
||||
|
||||
def _update_tag_sets(
|
||||
site: Path,
|
||||
lang: str,
|
||||
filecount: dict[str, dict[str, int]],
|
||||
files_by_tag: dict[str, list[Path]],
|
||||
tags_by_lang: dict[str, dict[str, str]],
|
||||
) -> None:
|
||||
"""
|
||||
Update the .tags.??.xml tagset xmls for a given tag
|
||||
"""
|
||||
# Add uout toplevel element
|
||||
page = etree.Element("tagset")
|
||||
|
||||
# Add the subelements
|
||||
version = etree.SubElement(page, "version")
|
||||
version.text = "1"
|
||||
for section in ["news", "events"]:
|
||||
for tag in files_by_tag:
|
||||
count = filecount[section][tag]
|
||||
label = (
|
||||
tags_by_lang[lang][tag]
|
||||
if keys_exists(tags_by_lang, lang, tag)
|
||||
and tags_by_lang[lang][tag] is not None
|
||||
else tags_by_lang["en"][tag]
|
||||
if keys_exists(tags_by_lang, "en", tag)
|
||||
and tags_by_lang["en"][tag] is not None
|
||||
else tag
|
||||
)
|
||||
if count > 0:
|
||||
etree.SubElement(
|
||||
page, "tag", section=section, key=tag, count=str(count)
|
||||
).text = label
|
||||
update_if_changed(
|
||||
Path(f"{site}/tags/.tags.{lang}.xml"),
|
||||
etree.tostring(page, encoding="utf-8").decode("utf-8"),
|
||||
)
|
||||
|
||||
|
||||
def update_tags(languages: list[str], pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
Update Tag pages, xmllists and xmls
|
||||
|
||||
Creates/update the following files:
|
||||
|
||||
* */tags/tagged-<tags>.en.xhtml for each tag used. Apart from being
|
||||
automatically created, these are regular source files for HTML pages, and
|
||||
in phase 2 are built into pages listing all news items and events for a
|
||||
tag.
|
||||
|
||||
* */tags/.tags.??.xml with a list of the tags used.
|
||||
|
||||
Changing or removing tags in XML files is also considered, in which case a
|
||||
file is removed from the .xmllist files.
|
||||
|
||||
When a tag has been removed from the last XML file where it has been used,
|
||||
the tagged-* are correctly deleted.
|
||||
"""
|
||||
for site in filter(
|
||||
lambda path: path.joinpath("tags").exists(),
|
||||
Path(".").glob("?*.??*"),
|
||||
):
|
||||
logger.info(f"Updating tags for {site}")
|
||||
# Create a complete and current map of which tag is used in which files
|
||||
files_by_tag = {}
|
||||
tags_by_lang = {}
|
||||
# Fill out files_by_tag and tags_by_lang
|
||||
for file in filter(
|
||||
lambda file:
|
||||
# Not in tags dir of a site
|
||||
site.joinpath("tags") not in file.parents,
|
||||
site.glob("**/*.xml"),
|
||||
):
|
||||
for tag in etree.parse(file).xpath("//tag"):
|
||||
# Get the key attribute, and filter out some invalid chars
|
||||
key = (
|
||||
tag.get("key")
|
||||
.replace("/", "-")
|
||||
.replace(" ", "-")
|
||||
.replace(":", "-")
|
||||
.strip()
|
||||
)
|
||||
# Get the label, and strip it.
|
||||
label = (
|
||||
escape(tag.text.strip()) if tag.text and tag.text.strip() else None
|
||||
)
|
||||
|
||||
# Load into the dicts
|
||||
if key not in files_by_tag:
|
||||
files_by_tag[key] = set()
|
||||
files_by_tag[key].add(get_basepath(file))
|
||||
lang = lang_from_filename(file)
|
||||
if lang not in tags_by_lang:
|
||||
tags_by_lang[lang] = {}
|
||||
tags_by_lang[lang][key] = (
|
||||
tags_by_lang[lang][key]
|
||||
if key in tags_by_lang[lang] and tags_by_lang[lang][key]
|
||||
else label
|
||||
)
|
||||
# Sort dicts to ensure that they are stable between runs
|
||||
files_by_tag = sort_dict(files_by_tag)
|
||||
for tag in files_by_tag:
|
||||
files_by_tag[tag] = sorted(files_by_tag[tag])
|
||||
tags_by_lang = sort_dict(tags_by_lang)
|
||||
for lang in tags_by_lang:
|
||||
tags_by_lang[lang] = sort_dict(tags_by_lang[lang])
|
||||
|
||||
logger.debug("Updating tag pages")
|
||||
pool.starmap(
|
||||
_update_tag_pages,
|
||||
map(lambda tag: (site, tag, languages), files_by_tag.keys()),
|
||||
)
|
||||
|
||||
logger.debug("Updating tag lists")
|
||||
pool.starmap(
|
||||
update_if_changed,
|
||||
map(
|
||||
lambda tag: (
|
||||
Path(f"{site}/tags/.tagged-{tag}.xmllist"),
|
||||
("\n".join(map(lambda file: str(file), files_by_tag[tag])) + "\n"),
|
||||
),
|
||||
files_by_tag.keys(),
|
||||
),
|
||||
)
|
||||
|
||||
logger.debug("Updating tag sets")
|
||||
# Get count of files with each tag in each section
|
||||
filecount = {}
|
||||
for section in ["news", "events"]:
|
||||
filecount[section] = {}
|
||||
for tag in files_by_tag:
|
||||
filecount[section][tag] = len(
|
||||
list(
|
||||
filter(
|
||||
lambda path: section in str(path.parent),
|
||||
files_by_tag[tag],
|
||||
)
|
||||
)
|
||||
)
|
||||
pool.starmap(
|
||||
_update_tag_sets,
|
||||
map(
|
||||
lambda lang: (site, lang, filecount, files_by_tag, tags_by_lang),
|
||||
filter(lambda lang: lang in languages, tags_by_lang.keys()),
|
||||
),
|
||||
)
|
||||
164
build/phase1/update_xmllists.py
Executable file
164
build/phase1/update_xmllists.py
Executable file
@ -0,0 +1,164 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import datetime
|
||||
import fnmatch
|
||||
import logging
|
||||
import multiprocessing
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import lxml.etree as etree
|
||||
|
||||
from build.lib.misc import (
|
||||
get_basepath,
|
||||
lang_from_filename,
|
||||
touch_if_newer_dep,
|
||||
update_if_changed,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _update_for_base(
|
||||
base: Path, all_xml: set[Path], nextyear: str, thisyear: str, lastyear: str
|
||||
) -> None:
|
||||
"""
|
||||
Update the xmllist for a given base file
|
||||
"""
|
||||
matching_files = set()
|
||||
# If sources exist
|
||||
if base.with_suffix(".sources").exists():
|
||||
# Load every file that matches the pattern
|
||||
# If a tag is included in the pattern, the file must contain that tag
|
||||
with base.with_suffix(".sources").open(mode="r") as file:
|
||||
for line in file:
|
||||
pattern = (
|
||||
re.sub(r":\[.*\]$", "*", line)
|
||||
.replace("$nextyear", nextyear)
|
||||
.replace("$thisyear", thisyear)
|
||||
.replace("$lastyear", lastyear)
|
||||
.strip()
|
||||
)
|
||||
if len(pattern) <= 0:
|
||||
logger.debug("Pattern too short, continue!")
|
||||
continue
|
||||
tag = (
|
||||
re.match(r":\[(.*)\]$", line).group().strip()
|
||||
if re.match(r":\[(.*)\]$", line)
|
||||
else ""
|
||||
)
|
||||
for line in filter(
|
||||
lambda line:
|
||||
# Matches glob pattern
|
||||
fnmatch.fnmatchcase(str(line), pattern)
|
||||
# contains tag if tag in pattern
|
||||
and (
|
||||
etree.parse(file).find(f"//tag[@key='{tag}']")
|
||||
if tag != ""
|
||||
else True
|
||||
)
|
||||
# Not just matching an empty line
|
||||
and len(str(line)) > 0,
|
||||
all_xml,
|
||||
):
|
||||
matching_files.add(str(line))
|
||||
|
||||
for file in Path("").glob(f"{base}.??.xhtml"):
|
||||
xslt_root = etree.parse(file)
|
||||
for module in xslt_root.xpath("//module"):
|
||||
matching_files.add(f"global/data/modules/{module.get('id')}".strip())
|
||||
matching_files = sorted(matching_files)
|
||||
update_if_changed(
|
||||
Path(f"{base.parent}/.{base.name}.xmllist"),
|
||||
("\n".join(matching_files) + "\n") if matching_files else "",
|
||||
)
|
||||
|
||||
|
||||
def _update_module_xmllists(languages: list[str], pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
Update .xmllist files for .sources and .xhtml containing <module>s
|
||||
"""
|
||||
logger.info("Updating XML lists")
|
||||
# Store current dir
|
||||
for site in filter(lambda path: path.is_dir(), Path(".").glob("?*.??*")):
|
||||
logger.info(f"Updating xmllists for {site}")
|
||||
# Get all the bases and stuff before multithreading the update bit
|
||||
all_xml = set(
|
||||
map(
|
||||
lambda path: get_basepath(path),
|
||||
filter(
|
||||
lambda path: lang_from_filename(path) in languages,
|
||||
site.glob("**/*.*.xml"),
|
||||
),
|
||||
)
|
||||
)
|
||||
source_bases = set(
|
||||
map(
|
||||
lambda path: path.with_suffix(""),
|
||||
site.glob("**/*.sources"),
|
||||
)
|
||||
)
|
||||
module_bases = set(
|
||||
map(
|
||||
lambda path: get_basepath(path),
|
||||
filter(
|
||||
lambda path: lang_from_filename(path) in languages
|
||||
and etree.parse(path).xpath("//module"),
|
||||
site.glob("**/*.*.xhtml"),
|
||||
),
|
||||
)
|
||||
)
|
||||
all_bases = source_bases | module_bases
|
||||
nextyear = str(datetime.datetime.today().year + 1)
|
||||
thisyear = str(datetime.datetime.today().year)
|
||||
lastyear = str(datetime.datetime.today().year - 1)
|
||||
pool.starmap(
|
||||
_update_for_base,
|
||||
map(lambda base: (base, all_xml, nextyear, thisyear, lastyear), all_bases),
|
||||
)
|
||||
|
||||
|
||||
def _check_xmllist_deps(file: Path) -> None:
|
||||
"""
|
||||
If any of the sources in an xmllist are newer than it, touch the xmllist
|
||||
"""
|
||||
xmls = set()
|
||||
with file.open(mode="r") as fileobj:
|
||||
for line in fileobj:
|
||||
for newfile in Path("").glob(line + ".??.xml"):
|
||||
xmls.add(newfile)
|
||||
touch_if_newer_dep(file, list(xmls))
|
||||
|
||||
|
||||
def _touch_xmllists_with_updated_deps(
|
||||
languages: list[str], pool: multiprocessing.Pool
|
||||
) -> None:
|
||||
"""
|
||||
Touch all .xmllist files where one of the contained files has changed
|
||||
"""
|
||||
logger.info("Checking contents of XML lists")
|
||||
pool.map(_check_xmllist_deps, Path("").glob("./**/.*.xmllist"))
|
||||
|
||||
|
||||
def update_xmllists(languages: list[str], pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
Update XML filelists (*.xmllist)
|
||||
|
||||
Creates/update the following files:
|
||||
|
||||
* <dir>/.<base>.xmllist for each <dir>/<base>.sources as well as for each
|
||||
fsfe.org/tags/tagged-<tags>.en.xhtml. These files are used in phase 2 to include the
|
||||
correct XML files when generating the HTML pages. It is taken care that
|
||||
these files are only updated whenever their content actually changes, so
|
||||
they can serve as a prerequisite in the phase 2 Makefile.
|
||||
|
||||
Changing or removing tags in XML files is also considered, in which case a
|
||||
file is removed from the .xmllist files.
|
||||
|
||||
When a tag has been removed from the last XML file where it has been used,
|
||||
the tagged-* are correctly deleted.
|
||||
"""
|
||||
_update_module_xmllists(languages, pool)
|
||||
_touch_xmllists_with_updated_deps(languages, pool)
|
||||
6
build/phase2/__init__.py
Normal file
6
build/phase2/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
56
build/phase2/copy_files.py
Normal file
56
build/phase2/copy_files.py
Normal file
@ -0,0 +1,56 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _copy_file(target: Path, source_file: Path) -> None:
|
||||
target_file = target.joinpath(source_file)
|
||||
if (
|
||||
not target_file.exists()
|
||||
or source_file.stat().st_mtime > target_file.stat().st_mtime
|
||||
):
|
||||
logger.debug(f"Copying {source_file} to {target_file}")
|
||||
target_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
target_file.write_bytes(source_file.read_bytes())
|
||||
|
||||
|
||||
def copy_files(pool: multiprocessing.Pool, target: Path) -> None:
|
||||
"""
|
||||
Copy images, docments etc
|
||||
"""
|
||||
logger.info("Copying over media and misc files")
|
||||
pool.starmap(
|
||||
_copy_file,
|
||||
map(
|
||||
lambda file: (target, file),
|
||||
list(
|
||||
filter(
|
||||
lambda path: path.is_file()
|
||||
and path.suffix
|
||||
not in [
|
||||
".md",
|
||||
".yml",
|
||||
".gitignore",
|
||||
".sources",
|
||||
".xmllist",
|
||||
".xhtml",
|
||||
".xsl",
|
||||
".xml",
|
||||
".less",
|
||||
".py",
|
||||
".pyc",
|
||||
]
|
||||
and path.name not in ["Makefile"],
|
||||
Path("").glob("*?.?*/**/*"),
|
||||
)
|
||||
)
|
||||
# Special case hard code pass over orde items xml required by cgi script
|
||||
+ list(Path("").glob("*?.?*/order/data/items.en.xml")),
|
||||
),
|
||||
)
|
||||
33
build/phase2/create_index_symlinks.py
Normal file
33
build/phase2/create_index_symlinks.py
Normal file
@ -0,0 +1,33 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
from build.lib.misc import get_basename
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _do_symlinking(target: Path) -> None:
|
||||
source = target.parent.joinpath(
|
||||
f"index{target.with_suffix('').suffix}{target.suffix}"
|
||||
)
|
||||
if not source.exists():
|
||||
source.symlink_to(target.relative_to(source.parent))
|
||||
|
||||
|
||||
def create_index_symlinks(pool: multiprocessing.Pool, target: Path) -> None:
|
||||
"""
|
||||
Create index.* symlinks
|
||||
"""
|
||||
logger.info("Creating index symlinks")
|
||||
pool.map(
|
||||
_do_symlinking,
|
||||
filter(
|
||||
lambda path: get_basename(path) == path.parent.name,
|
||||
target.glob("**/*.??.html"),
|
||||
),
|
||||
)
|
||||
26
build/phase2/create_language_symlinks.py
Normal file
26
build/phase2/create_language_symlinks.py
Normal file
@ -0,0 +1,26 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _do_symlinking(target: Path) -> None:
|
||||
source = target.with_suffix("").with_suffix(f".html{target.with_suffix('').suffix}")
|
||||
if not source.exists():
|
||||
source.symlink_to(target.relative_to(source.parent))
|
||||
|
||||
|
||||
def create_language_symlinks(pool: multiprocessing.Pool, target: Path) -> None:
|
||||
"""
|
||||
Create symlinks from file.<lang>.html to file.html.<lang>
|
||||
"""
|
||||
logger.info("Creating language symlinks")
|
||||
pool.map(
|
||||
_do_symlinking,
|
||||
target.glob("**/*.??.html"),
|
||||
)
|
||||
68
build/phase2/process_rss_ics_files.py
Normal file
68
build/phase2/process_rss_ics_files.py
Normal file
@ -0,0 +1,68 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
from build.lib.misc import get_basepath
|
||||
from build.lib.process_file import process_file
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _process_stylesheet(languages: list[str], target: Path, source_xsl: Path) -> None:
|
||||
base_file = get_basepath(source_xsl)
|
||||
destination_base = target.joinpath(base_file)
|
||||
for lang in languages:
|
||||
target_file = destination_base.with_suffix(
|
||||
f".{lang}{source_xsl.with_suffix('').suffix}"
|
||||
)
|
||||
source_xhtml = base_file.with_suffix(f".{lang}.xhtml")
|
||||
if not target_file.exists() or any(
|
||||
# If any source file is newer than the file to be generated
|
||||
map(
|
||||
lambda file: (
|
||||
file.exists() and file.stat().st_mtime > target_file.stat().st_mtime
|
||||
),
|
||||
[
|
||||
(
|
||||
source_xhtml
|
||||
if source_xhtml.exists()
|
||||
else base_file.with_suffix(".en.xhtml")
|
||||
),
|
||||
source_xsl,
|
||||
Path(f"global/data/texts/.texts.{lang}.xml"),
|
||||
Path("global/data/texts/texts.en.xml"),
|
||||
],
|
||||
)
|
||||
):
|
||||
logger.debug(f"Building {target_file}")
|
||||
result = process_file(source_xhtml, source_xsl)
|
||||
target_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
target_file.write_text(result)
|
||||
|
||||
|
||||
def process_rss_ics_files(
|
||||
languages: list[str], pool: multiprocessing.Pool, target: Path
|
||||
) -> None:
|
||||
"""
|
||||
Build .rss files from .xhtml sources
|
||||
"""
|
||||
logger.info("Processing rss files")
|
||||
pool.starmap(
|
||||
_process_stylesheet,
|
||||
map(
|
||||
lambda source_xsl: (languages, target, source_xsl),
|
||||
Path("").glob("*?.?*/**/*.rss.xsl"),
|
||||
),
|
||||
)
|
||||
logger.info("Processing ics files")
|
||||
pool.starmap(
|
||||
_process_stylesheet,
|
||||
map(
|
||||
lambda source_xsl: (languages, target, source_xsl),
|
||||
Path("").glob("*?.?*/**/*.ics.xsl"),
|
||||
),
|
||||
)
|
||||
69
build/phase2/process_xhtml_files.py
Normal file
69
build/phase2/process_xhtml_files.py
Normal file
@ -0,0 +1,69 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
from build.lib.process_file import process_file
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _process_dir(languages: list[str], target: Path, dir: Path) -> None:
|
||||
for basename in set(map(lambda path: path.with_suffix(""), dir.glob("*.??.xhtml"))):
|
||||
for lang in languages:
|
||||
source_file = basename.with_suffix(f".{lang}.xhtml")
|
||||
target_file = target.joinpath(source_file).with_suffix(".html")
|
||||
processor = (
|
||||
basename.with_suffix(".xsl")
|
||||
if basename.with_suffix(".xsl").exists()
|
||||
else basename.parent.joinpath(".default.xsl")
|
||||
)
|
||||
if not target_file.exists() or any(
|
||||
# If any source file is newer than the file to be generated
|
||||
# If the file does not exist to
|
||||
map(
|
||||
lambda file: (
|
||||
file.exists()
|
||||
and file.stat().st_mtime > target_file.stat().st_mtime
|
||||
),
|
||||
[
|
||||
(
|
||||
source_file
|
||||
if source_file.exists()
|
||||
else basename.with_suffix(".en.xhtml")
|
||||
),
|
||||
processor,
|
||||
target_file.with_suffix("").with_suffix(".xmllist"),
|
||||
Path(f"global/data/texts/.texts.{lang}.xml"),
|
||||
Path(f"global/data/topbanner/.topbanner.{lang}.xml"),
|
||||
Path("global/data/texts/texts.en.xml"),
|
||||
],
|
||||
)
|
||||
):
|
||||
logger.debug(f"Building {target_file}")
|
||||
result = process_file(source_file, processor)
|
||||
target_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
target_file.write_text(result)
|
||||
|
||||
|
||||
def process_xhtml_files(
|
||||
languages: list[str], pool: multiprocessing.Pool, target: Path
|
||||
) -> None:
|
||||
"""
|
||||
Build .html files from .xhtml sources
|
||||
"""
|
||||
# TODO
|
||||
# It should be possible to upgrade this and process_rss_ics files such that only one functions is needed
|
||||
# Also for performance it would be better to iterate by processor xls, and parse it only once and pass the xsl object to called function.
|
||||
logger.info("Processing xhtml files")
|
||||
|
||||
pool.starmap(
|
||||
_process_dir,
|
||||
map(
|
||||
lambda dir: (languages, target, dir),
|
||||
set(map(lambda path: path.parent, Path("").glob("*?.?*/**/*.*.xhtml"))),
|
||||
),
|
||||
)
|
||||
30
build/phase2/run.py
Normal file
30
build/phase2/run.py
Normal file
@ -0,0 +1,30 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# script for FSFE website build, phase 2
|
||||
# -----------------------------------------------------------------------------
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
from .copy_files import copy_files
|
||||
from .create_index_symlinks import create_index_symlinks
|
||||
from .create_language_symlinks import create_language_symlinks
|
||||
from .process_rss_ics_files import process_rss_ics_files
|
||||
from .process_xhtml_files import process_xhtml_files
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def phase2_run(languages: list[str], pool: multiprocessing.Pool, target: Path):
|
||||
"""
|
||||
Run all the necessary sub functions for phase2.
|
||||
"""
|
||||
logger.info("Starting Phase 2 - Generating output")
|
||||
process_xhtml_files(languages, pool, target)
|
||||
create_index_symlinks(pool, target)
|
||||
create_language_symlinks(pool, target)
|
||||
process_rss_ics_files(languages, pool, target)
|
||||
copy_files(pool, target)
|
||||
6
build/phase3/__init__.py
Normal file
6
build/phase3/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
39
build/phase3/serve_websites.py
Executable file
39
build/phase3/serve_websites.py
Executable file
@ -0,0 +1,39 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import http.server
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import socketserver
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _run_webserver(path: str, port: int) -> None:
|
||||
"""
|
||||
Given a path as a string and a port it will
|
||||
serve that dir on that localhost:port for forever.
|
||||
"""
|
||||
os.chdir(path)
|
||||
Handler = http.server.CGIHTTPRequestHandler
|
||||
|
||||
with socketserver.TCPServer(("", port), Handler) as httpd:
|
||||
httpd.serve_forever()
|
||||
|
||||
|
||||
def serve_websites(serve_dir: str, base_port: int, increment_number: int) -> None:
|
||||
"""
|
||||
Takes a target directory, a base port and a number to increment port by per dir
|
||||
It then serves all directories over http on localhost
|
||||
"""
|
||||
dirs = sorted(list(filter(lambda path: path.is_dir(), Path(serve_dir).iterdir())))
|
||||
serves = []
|
||||
for dir in dirs:
|
||||
port = base_port + (increment_number * dirs.index(dir))
|
||||
logging.info(f"{dir.name} served at http://127.0.0.1:{port}")
|
||||
serves.append((str(dir), port))
|
||||
with multiprocessing.Pool(len(serves)) as pool:
|
||||
pool.starmap(_run_webserver, serves)
|
||||
49
build/phase3/stage_to_target.py
Normal file
49
build/phase3/stage_to_target.py
Normal file
@ -0,0 +1,49 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
|
||||
from build.lib.misc import run_command
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _rsync(stagedir: Path, target: str, port: int) -> None:
|
||||
run_command(
|
||||
[
|
||||
"rsync",
|
||||
"-av",
|
||||
"--copy-unsafe-links",
|
||||
"--del",
|
||||
str(stagedir) + "/",
|
||||
target,
|
||||
]
|
||||
# Use ssh with a command such that it does not worry about fingerprints, as every connection is a new one basically
|
||||
# Also specify the sshport, and only load this sshconfig if required
|
||||
+ (
|
||||
["-e", f"ssh -o StrictHostKeyChecking=accept-new -p {port}"]
|
||||
if ":" in target
|
||||
else []
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def stage_to_target(stagedir: Path, targets: str, pool: multiprocessing.Pool) -> None:
|
||||
"""
|
||||
Use a multithreaded rsync to copy the stage dir to all targets.
|
||||
"""
|
||||
logger.info("Rsyncing from stage dir to target dir(s)")
|
||||
pool.starmap(
|
||||
_rsync,
|
||||
map(
|
||||
lambda target: (
|
||||
stagedir,
|
||||
(target if "?" not in target else target.split("?")[0]),
|
||||
(int(target.split("?")[1]) if "?" in target else 22),
|
||||
),
|
||||
targets.split(","),
|
||||
),
|
||||
)
|
||||
@ -1,11 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
basedir="${0%/*}/.."
|
||||
[ -z "$inc_processor" ] && . "$basedir/build/processor.sh"
|
||||
|
||||
. "$basedir/build/arguments.sh"
|
||||
|
||||
case "$command" in
|
||||
process_file) process_file "$workfile" "$processor" ;;
|
||||
*) die "Unrecognised command or no command given" ;;
|
||||
esac
|
||||
@ -1,56 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
inc_processor=true
|
||||
[ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh"
|
||||
[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh"
|
||||
|
||||
process_file() {
|
||||
infile="$1"
|
||||
processor="$2"
|
||||
|
||||
shortname=$(get_shortname "$infile")
|
||||
lang=$(get_language "$infile")
|
||||
|
||||
if [ -z "${processor}" ]; then
|
||||
if [ -f "${shortname}.xsl" ]; then
|
||||
processor="${shortname}.xsl"
|
||||
else
|
||||
# Actually use the symlink target, so the relative includes are searched
|
||||
# in the correct directory.
|
||||
processor="$(realpath "${shortname%/*}/.default.xsl")"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Make sure that the following pipe exits with a nonzero exit code if *any*
|
||||
# of the commands fails.
|
||||
set -o pipefail
|
||||
|
||||
# The sed command of death below does the following:
|
||||
# 1. Remove https://fsfe.org (or https://test.fsfe.org) from the start of all
|
||||
# links
|
||||
# 2. Change links from /foo/bar.html into /foo/bar.xx.html
|
||||
# 3. Change links from foo/bar.html into foo/bar.xx.html
|
||||
# 4. Same for .rss and .ics links
|
||||
# 5. Change links from /foo/bar/ into /foo/bar/index.xx.html
|
||||
# 6. Change links from foo/bar/ into foo/bar/index.xx.html
|
||||
# ... where xx is the language code.
|
||||
# Everything is duplicated to allow for the href attribute to be enclosed in
|
||||
# single or double quotes.
|
||||
# I am strongly convinced that there must be a less obfuscated way of doing
|
||||
# this. --Reinhard
|
||||
|
||||
build_xmlstream "$shortname" "$lang" |
|
||||
xsltproc --stringparam "build-env" "${build_env}" "$processor" - |
|
||||
sed -r ':X; N; $!bX;
|
||||
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="((https?:)?//[^"]*)";<\1\2 href="#== norewrite ==\3";gI
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="([^#"])([^"]*/)?([^\./"]*\.)(html|rss|ics)(#[^"]*)?";<\1\2 href="\3\4\5'"$lang"'.\6\7";gI
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="([^#"]*/)(#[^"]*)?";<\1\2 href="\3index.'"$lang"'.html\4";gI
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="#== norewrite ==((https?:)?//[^"]*)";<\1\2 href="\3";gI
|
||||
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href='\''((https?:)?//[^'\'']*)'\'';<\1\2 href='\''#== norewrite ==\3'\'';gI
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href='\''([^#'\''])([^'\'']*/)?([^\./'\'']*\.)(html|rss|ics)(#[^'\'']*)?'\'';<\1\2 href='\''\3\4\5'"$lang"'.\6\7'\'';gI
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href='\''([^#'\'']*/)(#[^'\'']*)?'\'';<\1\2 href='\''\3index.'"$lang"'.html\4'\'';gI
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href='\''#== norewrite ==((https?:)?//[^'\'']*)'\'';<\1\2 href='\''\3'\'';gI
|
||||
'
|
||||
}
|
||||
@ -1,125 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
inc_scaffold=true
|
||||
|
||||
get_version() {
|
||||
version=$(xsltproc $basedir/build/xslt/get_version.xsl $1)
|
||||
echo ${version:-0}
|
||||
}
|
||||
|
||||
include_xml() {
|
||||
# include second level elements of a given XML file
|
||||
# this emulates the behaviour of the original
|
||||
# build script which wasn't able to load top
|
||||
# level elements from any file
|
||||
if [ -f "$1" ]; then
|
||||
# Remove <version> because the filename attribute would otherwise be added
|
||||
# to this element instead of the actual content element.
|
||||
sed 's;<version>.*</version>;;' "$1" |
|
||||
sed -r ':X; $bY; N; bX; :Y;
|
||||
s:<(\?[xX][mM][lL]|!DOCTYPE)[[:space:]]+[^>]+>::g
|
||||
s:<[^!][^>]*>::;
|
||||
s:</[^>]*>([^<]*((<[^>]+/>|<!([^>]|<[^>]*>)*>|<\?[^>]+>)[^<]*)*)?$:\1:;'
|
||||
fi
|
||||
}
|
||||
|
||||
get_attributes() {
|
||||
# get attributes of top level element in a given
|
||||
# XHTML file
|
||||
sed -rn ':X; N; $!bX;
|
||||
s;^.*<[\n\t\r ]*([xX]|[xX]?[hH][tT])[mM][lL][\n\t\r ]+([^>]*)>.*$;\2;p' "$1"
|
||||
}
|
||||
|
||||
list_langs() {
|
||||
# list all languages a file exists in by globbing up
|
||||
# the shortname (i.e. file path with file ending omitted)
|
||||
# output is readily formatted for inclusion
|
||||
# in xml stream
|
||||
for file in "${1}".[a-z][a-z].xhtml; do
|
||||
language="${file: -8:2}"
|
||||
text="$(echo -n $(cat "${basedir}/global/languages/${language}"))"
|
||||
echo "<tr id=\"${language}\">${text}</tr>"
|
||||
done
|
||||
}
|
||||
|
||||
auto_sources() {
|
||||
# import elements from source files, add file name
|
||||
# attribute to first element included from each file
|
||||
shortname="$1"
|
||||
lang="$2"
|
||||
|
||||
list_file="$(dirname ${shortname})/.$(basename ${shortname}).xmllist"
|
||||
|
||||
if [ -f "${list_file}" ]; then
|
||||
cat "${list_file}" | while read path; do
|
||||
base="$(basename ${path})"
|
||||
if [ -f "${basedir}/${path}.${lang}.xml" ]; then
|
||||
printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.${lang}.xml")"
|
||||
elif [ -f "${basedir}/${path}.en.xml" ]; then
|
||||
printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.en.xml")"
|
||||
fi
|
||||
done |
|
||||
sed -r ':X; N; $!bX;
|
||||
s;\n### (filename="[^\n"]+") ###\n[^<]*(<![^>]+>[^<]*)*(<([^/>]+/)*([^/>]+))(/?>);\2\3 \1\6;g;'
|
||||
fi
|
||||
}
|
||||
|
||||
build_xmlstream() {
|
||||
# assemble the xml stream for feeding into xsltproc
|
||||
# the expected shortname and language flag indicate
|
||||
# a single xhtml page to be built
|
||||
shortname="$1"
|
||||
lang="$2"
|
||||
|
||||
olang="$(echo "${shortname}".[a-z][a-z].xhtml "${shortname}".[e]n.xhtml | sed -rn 's;^.*\.([a-z]{2})\.xhtml.*$;\1;p')"
|
||||
dirname="${shortname%/*}/"
|
||||
topbanner_xml="$basedir/global/data/topbanner/.topbanner.${lang}.xml"
|
||||
texts_xml="$basedir/global/data/texts/.texts.${lang}.xml"
|
||||
date="$(date +%Y-%m-%d)"
|
||||
time="$(date +%H:%M:%S)"
|
||||
|
||||
if [ -f "${shortname}.${lang}.xhtml" ]; then
|
||||
act_lang="$lang"
|
||||
translation_state="up-to-date"
|
||||
[ $(get_version "${shortname}.${olang}.xhtml") -gt $(get_version "${shortname}.${lang}.xhtml") ] && translation_state="outdated"
|
||||
[ $(($(get_version "${shortname}.${olang}.xhtml") - 3)) -gt $(get_version "${shortname}.${lang}.xhtml") ] && act_lang="$olang" && translation_state="very-outdated"
|
||||
else
|
||||
act_lang="$olang"
|
||||
translation_state="untranslated"
|
||||
fi
|
||||
|
||||
infile="${shortname}.${act_lang}.xhtml"
|
||||
|
||||
cat <<-EOF
|
||||
<buildinfo
|
||||
date="$date"
|
||||
original="$olang"
|
||||
filename="/${shortname#"$basedir"/}"
|
||||
fileurl="/${shortname#"$basedir"/*/}"
|
||||
dirname="/${dirname#"$basedir"/}"
|
||||
language="$lang"
|
||||
translation_state="$translation_state"
|
||||
>
|
||||
|
||||
<trlist>
|
||||
$(list_langs "$shortname")
|
||||
</trlist>
|
||||
|
||||
<topbanner>$(include_xml "$topbanner_xml")</topbanner>
|
||||
<textsetbackup>$(include_xml "$basedir/global/data/texts/texts.en.xml")</textsetbackup>
|
||||
<textset>$(include_xml "$texts_xml")</textset>
|
||||
|
||||
<document
|
||||
language="$act_lang"
|
||||
$(get_attributes "$infile")
|
||||
>
|
||||
<set>
|
||||
$(auto_sources "${shortname}" "$lang")
|
||||
</set>
|
||||
|
||||
$(include_xml "$infile")
|
||||
</document>
|
||||
|
||||
</buildinfo>
|
||||
EOF
|
||||
}
|
||||
@ -5,12 +5,6 @@
|
||||
<xsl:template name="body_scripts">
|
||||
<script src="{$urlprefix}/scripts/bootstrap-3.0.3.custom.js"></script>
|
||||
|
||||
<xsl:if test="$build-env = 'development'">
|
||||
<xsl:element name="script">
|
||||
<xsl:attribute name="src"><xsl:value-of select="$urlprefix"/>/scripts/less.min.js</xsl:attribute>
|
||||
</xsl:element>
|
||||
|
||||
</xsl:if>
|
||||
</xsl:template>
|
||||
|
||||
</xsl:stylesheet>
|
||||
|
||||
@ -54,45 +54,21 @@
|
||||
</xsl:element>
|
||||
|
||||
<xsl:choose>
|
||||
<xsl:when test="$build-env = 'development' and not(/buildinfo/document/@external)">
|
||||
<xsl:choose>
|
||||
<xsl:when test="$mode = 'valentine'">
|
||||
<xsl:element name="link">
|
||||
<xsl:attribute name="rel">stylesheet/less</xsl:attribute>
|
||||
<xsl:attribute name="media">all</xsl:attribute>
|
||||
<xsl:attribute name="href"><xsl:value-of select="$urlprefix"/>/look/valentine.less</xsl:attribute>
|
||||
<xsl:attribute name="type">text/css</xsl:attribute>
|
||||
</xsl:element>
|
||||
</xsl:when>
|
||||
<xsl:otherwise><!-- not valentine -->
|
||||
<xsl:element name="link">
|
||||
<xsl:attribute name="rel">stylesheet/less</xsl:attribute>
|
||||
<xsl:attribute name="media">all</xsl:attribute>
|
||||
<xsl:attribute name="href"><xsl:value-of select="$urlprefix"/>/look/fsfe.less</xsl:attribute>
|
||||
<xsl:attribute name="type">text/css</xsl:attribute>
|
||||
</xsl:element>
|
||||
</xsl:otherwise>
|
||||
</xsl:choose>
|
||||
<xsl:when test="$mode = 'valentine'">
|
||||
<xsl:element name="link">
|
||||
<xsl:attribute name="rel">stylesheet</xsl:attribute>
|
||||
<xsl:attribute name="media">all</xsl:attribute>
|
||||
<xsl:attribute name="href"><xsl:value-of select="$urlprefix"/>/look/valentine.min.css</xsl:attribute>
|
||||
<xsl:attribute name="type">text/css</xsl:attribute>
|
||||
</xsl:element>
|
||||
</xsl:when>
|
||||
<xsl:otherwise><!-- not development -->
|
||||
<xsl:choose>
|
||||
<xsl:when test="$mode = 'valentine'">
|
||||
<xsl:element name="link">
|
||||
<xsl:attribute name="rel">stylesheet</xsl:attribute>
|
||||
<xsl:attribute name="media">all</xsl:attribute>
|
||||
<xsl:attribute name="href"><xsl:value-of select="$urlprefix"/>/look/valentine.min.css</xsl:attribute>
|
||||
<xsl:attribute name="type">text/css</xsl:attribute>
|
||||
</xsl:element>
|
||||
</xsl:when>
|
||||
<xsl:otherwise><!-- not valentine -->
|
||||
<xsl:element name="link">
|
||||
<xsl:attribute name="rel">stylesheet</xsl:attribute>
|
||||
<xsl:attribute name="media">all</xsl:attribute>
|
||||
<xsl:attribute name="href"><xsl:value-of select="$urlprefix"/>/look/fsfe.min.css?20230215</xsl:attribute>
|
||||
<xsl:attribute name="type">text/css</xsl:attribute>
|
||||
</xsl:element>
|
||||
</xsl:otherwise>
|
||||
</xsl:choose>
|
||||
<xsl:otherwise><!-- not valentine -->
|
||||
<xsl:element name="link">
|
||||
<xsl:attribute name="rel">stylesheet</xsl:attribute>
|
||||
<xsl:attribute name="media">all</xsl:attribute>
|
||||
<xsl:attribute name="href"><xsl:value-of select="$urlprefix"/>/look/fsfe.min.css?20230215</xsl:attribute>
|
||||
<xsl:attribute name="type">text/css</xsl:attribute>
|
||||
</xsl:element>
|
||||
</xsl:otherwise>
|
||||
</xsl:choose>
|
||||
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!-- ====================================================================== -->
|
||||
<!-- XSL script to extract the <localmenu> dir attribute of an XML file -->
|
||||
<!-- ====================================================================== -->
|
||||
<!-- This XSL script outputs the "dir" attribute of the <localmenu> element -->
|
||||
<!-- of an XML file. It is used by the script tools/update_localmenus.sh. -->
|
||||
<!-- ====================================================================== -->
|
||||
|
||||
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
|
||||
<xsl:output method="text" encoding="UTF-8"/>
|
||||
|
||||
<xsl:template match="localmenu">
|
||||
<xsl:value-of select="@dir"/>
|
||||
</xsl:template>
|
||||
|
||||
<!-- Suppress output of text nodes, which would be the default -->
|
||||
<xsl:template match="text()"/>
|
||||
</xsl:stylesheet>
|
||||
@ -1,35 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!-- ====================================================================== -->
|
||||
<!-- XSL script to extract the <localmenu> element of an XML file -->
|
||||
<!-- ====================================================================== -->
|
||||
<!-- This XSL script outputs a line for the .localmenu.en.xml file from the -->
|
||||
<!-- <localmenu> element of an .xhtml file. It is used by the script -->
|
||||
<!-- tools/update_localmenus.sh. -->
|
||||
<!-- ====================================================================== -->
|
||||
|
||||
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
|
||||
<xsl:output method="text" encoding="UTF-8"/>
|
||||
|
||||
<xsl:template match="localmenu[@id]">
|
||||
<xsl:text> <localmenuitem set="</xsl:text>
|
||||
<xsl:choose>
|
||||
<xsl:when test="@set">
|
||||
<xsl:value-of select="@set"/>
|
||||
</xsl:when>
|
||||
<xsl:otherwise>
|
||||
<xsl:text>default</xsl:text>
|
||||
</xsl:otherwise>
|
||||
</xsl:choose>
|
||||
<xsl:text>" id="</xsl:text>
|
||||
<xsl:value-of select="@id"/>
|
||||
<xsl:text>" link="</xsl:text>
|
||||
<xsl:value-of select="$link"/>
|
||||
<xsl:text>"></xsl:text>
|
||||
<xsl:value-of select="normalize-space(node())"/>
|
||||
<xsl:text></localmenuitem></xsl:text>
|
||||
</xsl:template>
|
||||
|
||||
<!-- Suppress output of text nodes, which would be the default -->
|
||||
<xsl:template match="text()"/>
|
||||
</xsl:stylesheet>
|
||||
@ -1,25 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!-- ====================================================================== -->
|
||||
<!-- XSL script to extract the used modules from a .xhtml file -->
|
||||
<!-- ====================================================================== -->
|
||||
<!-- This XSL script processes all <module> elements of a .xhtml file and -->
|
||||
<!-- outputs the source files for these modules, separated by newlines. -->
|
||||
<!-- It is used by the script tools/update_xmllists.sh. -->
|
||||
<!-- ====================================================================== -->
|
||||
|
||||
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
|
||||
<xsl:output method="text" encoding="UTF-8"/>
|
||||
|
||||
<xsl:template match="module">
|
||||
<!-- Directory name -->
|
||||
<xsl:text>global/data/modules/</xsl:text>
|
||||
<!-- Filename = module id -->
|
||||
<xsl:value-of select="@id"/>
|
||||
<!-- Append a newline -->
|
||||
<xsl:text>
</xsl:text>
|
||||
</xsl:template>
|
||||
|
||||
<!-- Suppress output of text nodes, which would be the default -->
|
||||
<xsl:template match="text()"/>
|
||||
</xsl:stylesheet>
|
||||
@ -1,27 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<!-- ====================================================================== -->
|
||||
<!-- XSL script to extract the content of <tag> elements from an XML file -->
|
||||
<!-- ====================================================================== -->
|
||||
<!-- This XSL script processes all <tag> elements of an XML file and -->
|
||||
<!-- outputs the content of each of these elements, separated by newlines. -->
|
||||
<!-- It is used by the script tools/update_xmllists.sh. -->
|
||||
<!-- ====================================================================== -->
|
||||
|
||||
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
|
||||
<xsl:output method="text" encoding="UTF-8"/>
|
||||
|
||||
<xsl:template match="tag">
|
||||
<!-- Output tag name with some safeguarding against invalid characters -->
|
||||
<xsl:value-of select="translate(@key, ' /:', '---')"/>
|
||||
<!-- Output a blank -->
|
||||
<xsl:text> </xsl:text>
|
||||
<!-- Output tag label -->
|
||||
<xsl:value-of select="."/>
|
||||
<!-- Append a newline -->
|
||||
<xsl:text>
</xsl:text>
|
||||
</xsl:template>
|
||||
|
||||
<!-- Suppress output of text nodes, which would be the default -->
|
||||
<xsl:template match="text()"/>
|
||||
</xsl:stylesheet>
|
||||
@ -3,7 +3,6 @@ services:
|
||||
build: .
|
||||
image: fsfe-websites
|
||||
container_name: fsfe-websites
|
||||
command:
|
||||
ports:
|
||||
- 2000:2000
|
||||
- 2100:2100
|
||||
@ -11,5 +10,18 @@ services:
|
||||
- 2300:2300
|
||||
- 2400:2400
|
||||
- 2500:2500
|
||||
secrets:
|
||||
- KEY_PRIVATE
|
||||
- KEY_PASSWORD
|
||||
- GIT_TOKEN
|
||||
volumes:
|
||||
- ./:/fsfe-websites
|
||||
- website-cached:/website-cached
|
||||
volumes:
|
||||
website-cached:
|
||||
secrets:
|
||||
KEY_PRIVATE:
|
||||
environment: KEY_PRIVATE
|
||||
KEY_PASSWORD:
|
||||
environment: KEY_PASSWORD
|
||||
GIT_TOKEN:
|
||||
environment: GIT_TOKEN
|
||||
|
||||
46
entrypoint.sh
Normal file
46
entrypoint.sh
Normal file
@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
# Ran by dockerfile as entrypoint
|
||||
# Ran from the volume of the website source mounted at /website-source
|
||||
|
||||
# Load sshkeys
|
||||
if [ -f /run/secrets/KEY_PRIVATE ]; then
|
||||
# Start ssh-agent
|
||||
eval "$(ssh-agent)"
|
||||
|
||||
# Create config file with required keys
|
||||
mkdir -p ~/.ssh
|
||||
echo "AddKeysToAgent yes" > ~/.ssh/config
|
||||
# Tighten permissions to keep ssh-add happy
|
||||
chmod 400 /run/secrets/KEY_*
|
||||
PASSWORD="$(cat "/run/secrets/KEY_PASSWORD")"
|
||||
PRIVATE="$(cat "/run/secrets/KEY_PRIVATE")"
|
||||
# Really should be able to just read from the private path, but for some reason ssh-add fails when using the actual path
|
||||
# But works when you cat the path into another file and then load it
|
||||
# Or cat the file and pipe it in through stdin
|
||||
# Piping stdin to an expect command is quite complex, so we just make and remove a temporary key file.
|
||||
# Absolutely bizarre, and not quite ideal security wise
|
||||
echo "$PRIVATE" >/tmp/key
|
||||
chmod 600 /tmp/key
|
||||
|
||||
# Use our wrapper expect script to handle interactive input
|
||||
./exp.exp "$PASSWORD" ssh-add "/tmp/key"
|
||||
rm /tmp/key
|
||||
echo "SSH Key Loaded"
|
||||
else
|
||||
echo "Secret not defined!"
|
||||
fi
|
||||
|
||||
if [ -f /run/secrets/GIT_TOKEN ]; then
|
||||
export GIT_TOKEN="$(cat "/run/secrets/GIT_TOKEN")"
|
||||
fi
|
||||
|
||||
# Rsync files over, do not use the mtimes as they are wrong due to docker shenanigans
|
||||
# Use the .gitignore as a filter to not remove any files generated by previous runs
|
||||
rsync -rlpgoDz --delete --checksum --filter=':- .gitignore' ./ /website-cached/source
|
||||
|
||||
# Change to source repo
|
||||
cd /website-cached/source
|
||||
|
||||
# run build script expaning all args passed to this script
|
||||
python3 ./build.py "$@"
|
||||
12
exp.exp
Executable file
12
exp.exp
Executable file
@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env expect
|
||||
|
||||
set timeout 20
|
||||
|
||||
set cmd [lrange $argv 1 end]
|
||||
set password [lindex $argv 0]
|
||||
|
||||
eval spawn $cmd
|
||||
expect "Enter passphrase*:"
|
||||
send "$password\r"
|
||||
lassign [wait] pid spawn_id os_error actual_exit_code
|
||||
interact
|
||||
@ -1,39 +0,0 @@
|
||||
# -----------------------------------------------------------------------------
|
||||
# Makefile for FSFE website build, preparation for events subdirectory
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
.PHONY: all
|
||||
.SECONDEXPANSION:
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Copy event archive template to each of the years
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All years for which a subdirectory exists
|
||||
ARCH_YEARS := $(sort $(wildcard [0-9][0-9][0-9][0-9]))
|
||||
|
||||
# No archive for the current year
|
||||
ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS))
|
||||
|
||||
# ... and the year before
|
||||
ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS))
|
||||
|
||||
# Languages in which the template exists
|
||||
ARCH_LANGS := $(filter $(foreach lang,$(languages),.$(lang)),$(suffix $(basename $(wildcard archive-template.??.xhtml))))
|
||||
|
||||
# .xhtml files to generate
|
||||
ARCH_XHTML := $(foreach year,$(ARCH_YEARS),$(foreach lang,$(ARCH_LANGS),$(year)/index$(lang).xhtml))
|
||||
|
||||
all: $(ARCH_XHTML)
|
||||
$(ARCH_XHTML): %.xhtml: archive-template$$(suffix $$*).xhtml
|
||||
@echo "* Creating $@"
|
||||
@# $(dir $@) returns YYYY/, we abuse the slash for closing the sed command
|
||||
@sed 's/:YYYY:/$(dir $@)g' $< > $@
|
||||
|
||||
# .sources files to generate
|
||||
ARCH_SOURCES := $(foreach year,$(ARCH_YEARS),$(year)/index.sources)
|
||||
|
||||
all: $(ARCH_SOURCES)
|
||||
$(ARCH_SOURCES): %.sources:
|
||||
@echo "* Creating $@"
|
||||
@echo "fsfe.org/events/$(dir $@)event-*:[]\nfsfe.org/events/$(dir $@).event-*:[]\nfsfe.org/events/.localmenu:[]\n" > $@
|
||||
6
fsfe.org/events/__init__.py
Normal file
6
fsfe.org/events/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
51
fsfe.org/events/subdir.py
Normal file
51
fsfe.org/events/subdir.py
Normal file
@ -0,0 +1,51 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _gen_archive_index(working_dir: Path, languages: list[str], dir: Path):
|
||||
logger.debug(f"Operating on dir {dir}")
|
||||
for lang in languages:
|
||||
logger.debug(f"Operating on lang {lang}")
|
||||
template = working_dir.joinpath(f"archive-template.{lang}.xhtml")
|
||||
if template.exists():
|
||||
logger.debug("Template Exists!")
|
||||
content = template.read_text()
|
||||
content = content.replace(":YYYY:", dir.name)
|
||||
dir.joinpath(f"index.{lang}.xhtml").write_text(content)
|
||||
|
||||
|
||||
def _gen_index_sources(dir: Path):
|
||||
dir.joinpath("index.sources").write_text(
|
||||
dedent(
|
||||
f"""\
|
||||
{dir}/event-*:[]
|
||||
{dir}/.event-*:[]
|
||||
{dir.parent}/.localmenu:[]
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def run(languages: list[str], processes: int, working_dir: Path) -> None:
|
||||
"""
|
||||
preparation for news subdirectory
|
||||
"""
|
||||
with multiprocessing.Pool(processes) as pool:
|
||||
years = list(sorted(working_dir.glob("[0-9][0-9][0-9][0-9]")))
|
||||
# Copy news archive template to each of the years
|
||||
pool.starmap(
|
||||
_gen_archive_index,
|
||||
[(working_dir, languages, dir) for dir in years[:-2]],
|
||||
)
|
||||
logger.debug("Finished Archiving")
|
||||
# Generate index.sources for every year
|
||||
pool.map(_gen_index_sources, years)
|
||||
logger.debug("Finished generating sources")
|
||||
@ -1,74 +0,0 @@
|
||||
# -----------------------------------------------------------------------------
|
||||
# Makefile for FSFE website build, preparation for news subdirectory
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
.PHONY: all
|
||||
.SECONDEXPANSION:
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Copy news archive template to each of the years
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All years for which a subdirectory exists
|
||||
ARCH_YEARS := $(sort $(wildcard [0-9][0-9][0-9][0-9]))
|
||||
|
||||
# No archive for the current year
|
||||
ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS))
|
||||
|
||||
# ... and the year before
|
||||
ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS))
|
||||
|
||||
# Languages in which the template exists
|
||||
ARCH_LANGS := $(filter $(foreach lang,$(languages),.$(lang)),$(suffix $(basename $(wildcard archive-template.??.xhtml))))
|
||||
|
||||
# .xhtml files to generate
|
||||
ARCH_XHTML := $(foreach year,$(ARCH_YEARS),$(foreach lang,$(ARCH_LANGS),$(year)/index$(lang).xhtml))
|
||||
|
||||
all: $(ARCH_XHTML)
|
||||
$(ARCH_XHTML): %.xhtml: archive-template$$(suffix $$*).xhtml
|
||||
@echo "* Creating $@"
|
||||
@# $(dir $@) returns YYYY/, we abuse the slash for closing the sed command
|
||||
@sed 's/:YYYY:/$(dir $@)g' $< > $@
|
||||
|
||||
# .sources files to generate
|
||||
ARCH_SOURCES := $(foreach year,$(ARCH_YEARS),$(year)/index.sources)
|
||||
|
||||
all: $(ARCH_SOURCES)
|
||||
$(ARCH_SOURCES): %.sources:
|
||||
@echo "* Creating $@"
|
||||
@printf "fsfe.org/news/$(dir $@)news-*:[]\nfsfe.org/news/$(dir $@).news-*:[]\nfsfe.org/news/.localmenu:[]\n" > $@
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Remove generated .xml files where original .xhtml file does not exist anymore
|
||||
# -----------------------------------------------------------------------------
|
||||
# note the reversal of target <-> prerequisite relationship
|
||||
# make will execute thew command for all xhtml files (targets) that
|
||||
# do not exist, in doing so it will not make the target, but rather
|
||||
# remove the xml file that generated it
|
||||
|
||||
# All currently existing generated .xml files
|
||||
GENERATED_XML := $(wildcard */.*.xml)
|
||||
|
||||
# List of corresponding source files (foo/.bar.xx.xml -> foo/bar.xx.xhtml)
|
||||
GENERATED_XML_SOURCES := $(patsubst %.xml,%.xhtml,$(subst /.,/,$(GENERATED_XML)))
|
||||
|
||||
all: $(GENERATED_XML_SOURCES)
|
||||
%.xhtml:
|
||||
@echo '* Removing $(subst /,/.,$*).xml'
|
||||
@rm '$(subst /,/.,$*).xml'
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Generate .xml files from .xhtml files
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All existing .xhtml files
|
||||
XHTML := $(filter $(foreach lang,$(languages),%.$(lang).xhtml),$(shell ls */*.??.xhtml | xargs grep -l '<html newsdate'))
|
||||
|
||||
# List of .xml files to generate
|
||||
XML := $(patsubst %.xhtml,%.xml,$(subst /,/.,$(XHTML)))
|
||||
|
||||
all: $(XML)
|
||||
XMLSOURCE = $(patsubst %.xml,%.xhtml,$(subst /.,/,$@))
|
||||
%.xml: $$(XMLSOURCE) xhtml2xml.xsl
|
||||
@echo '* Generating $@'
|
||||
@xsltproc --stringparam link '/news/$(basename $(basename $<)).html' xhtml2xml.xsl '$<' > '$@'
|
||||
@ -1,12 +1,12 @@
|
||||
Adding news
|
||||
# Adding news
|
||||
===========
|
||||
|
||||
There are two ways to add news to the web pages:
|
||||
|
||||
**1**
|
||||
## 1
|
||||
Add an xml file with the following structure in the appropriate
|
||||
directory.
|
||||
|
||||
```xml
|
||||
<?xml version="1.0" encoding="UTF-8"?> (you can choose an other encoding)
|
||||
|
||||
<newsset>
|
||||
@ -18,25 +18,27 @@ directory.
|
||||
<link>link</link>
|
||||
</news>
|
||||
</newsset>
|
||||
```
|
||||
|
||||
|
||||
Put this file in the directory /news/this_year/
|
||||
There's a naming convention for these xml files:
|
||||
|
||||
```
|
||||
'news-'newsdate'-'counter'.'language_part'.xml'
|
||||
|
||||
```
|
||||
(eg: the English version of the first news file on the 4th November of
|
||||
2008 should be named news-20081104-01.en.xml and the file should go in
|
||||
the /news/2008/ directory)
|
||||
|
||||
**2**
|
||||
## 2
|
||||
Add an xhtml file in the appropriate directory.
|
||||
|
||||
Write an ordinary xhtml file. Add the newsdate="date" attribute in the
|
||||
xhtml tag. The first <p> element will be copied into the xml file.
|
||||
|
||||
(eg:
|
||||
|
||||
```xhtml
|
||||
<?xml versio ...
|
||||
|
||||
<html newsdate="2008-10-07" link="link" > (link attribute is optional)
|
||||
@ -54,7 +56,7 @@ xhtml tag. The first <p> element will be copied into the xml file.
|
||||
....
|
||||
|
||||
</html>
|
||||
|
||||
```
|
||||
The link in the generated xml file will default to the original
|
||||
page. If you want it to link to another page then you can use the
|
||||
link attribute in the html tag.
|
||||
6
fsfe.org/news/__init__.py
Normal file
6
fsfe.org/news/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
86
fsfe.org/news/subdir.py
Normal file
86
fsfe.org/news/subdir.py
Normal file
@ -0,0 +1,86 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import logging
|
||||
import multiprocessing
|
||||
from pathlib import Path
|
||||
from textwrap import dedent
|
||||
|
||||
import lxml.etree as etree
|
||||
|
||||
from build.lib.misc import lang_from_filename, update_if_changed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _gen_archive_index(working_dir: Path, languages: list[str], dir: Path):
|
||||
logger.debug(f"Operating on dir {dir}")
|
||||
for lang in languages:
|
||||
logger.debug(f"Operating on lang {lang}")
|
||||
template = working_dir.joinpath(f"archive-template.{lang}.xhtml")
|
||||
if template.exists():
|
||||
logger.debug("Template Exists!")
|
||||
content = template.read_text()
|
||||
content = content.replace(":YYYY:", dir.name)
|
||||
dir.joinpath(f"index.{lang}.xhtml").write_text(content)
|
||||
|
||||
|
||||
def _gen_index_sources(dir: Path):
|
||||
dir.joinpath("index.sources").write_text(
|
||||
dedent(
|
||||
f"""\
|
||||
{dir}/news-*:[]
|
||||
{dir}/.news-*:[]
|
||||
{dir.parent}/.localmenu:[]
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _gen_xml_files(working_dir: Path, file: Path):
|
||||
logger.debug(f"Transforming {file}")
|
||||
# Would be more efficient to pass this to the function, but this causes a pickling error, and the faq seems to indicate passing around these objects between threads causes issues
|
||||
# https://lxml.de/5.0/FAQ.html
|
||||
# So I guess we just have to take the performance hit.
|
||||
xslt_tree = etree.parse(working_dir.joinpath("xhtml2xml.xsl"))
|
||||
transform = etree.XSLT(xslt_tree)
|
||||
result = transform(
|
||||
etree.parse(file),
|
||||
link=f"'/news/{file.with_suffix('').with_suffix('.html').relative_to(file.parent.parent)}'",
|
||||
)
|
||||
update_if_changed(
|
||||
file.parent.joinpath(
|
||||
f".{file.with_suffix('').stem}{file.with_suffix('').suffix}.xml"
|
||||
),
|
||||
str(result),
|
||||
)
|
||||
|
||||
|
||||
def run(languages: list[str], processes: int, working_dir: Path) -> None:
|
||||
"""
|
||||
preparation for news subdirectory
|
||||
"""
|
||||
with multiprocessing.Pool(processes) as pool:
|
||||
years = list(sorted(working_dir.glob("[0-9][0-9][0-9][0-9]")))
|
||||
# Copy news archive template to each of the years
|
||||
pool.starmap(
|
||||
_gen_archive_index,
|
||||
[(working_dir, languages, dir) for dir in years[:-2]],
|
||||
)
|
||||
logger.debug("Finished Archiving")
|
||||
# Generate index.sources for every year
|
||||
pool.map(_gen_index_sources, years)
|
||||
logger.debug("Finished generating sources")
|
||||
pool.starmap(
|
||||
_gen_xml_files,
|
||||
[
|
||||
(working_dir, file)
|
||||
for file in filter(
|
||||
lambda path: lang_from_filename(path) in languages
|
||||
and etree.parse(path).xpath("//html[@newsdate]"),
|
||||
working_dir.glob("*/*.??.xhtml"),
|
||||
)
|
||||
],
|
||||
)
|
||||
logger.debug("Finished generating xml files")
|
||||
18
fsfe.org/scripts/less.min.js
vendored
18
fsfe.org/scripts/less.min.js
vendored
File diff suppressed because one or more lines are too long
@ -1 +1 @@
|
||||
العربيّة
|
||||
العربيّة
|
||||
|
||||
10
requirements.txt
Normal file
10
requirements.txt
Normal file
@ -0,0 +1,10 @@
|
||||
# XML parser
|
||||
lxml==5.3.2
|
||||
# For getting english language names of languages from two letter codes.
|
||||
python-iso639==2025.2.18
|
||||
# For stopwords for the search index
|
||||
nltk==3.9.1
|
||||
# For minification html css and js
|
||||
tdewolff-minify==2.20.37
|
||||
# For HTTP requests
|
||||
requests==2.32.3
|
||||
@ -1,77 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import http.server
|
||||
import multiprocessing
|
||||
import os
|
||||
import socketserver
|
||||
|
||||
|
||||
def run_webserver(path, PORT):
|
||||
os.chdir(path)
|
||||
Handler = http.server.SimpleHTTPRequestHandler
|
||||
httpd = socketserver.TCPServer(("0.0.0.0", PORT), Handler)
|
||||
|
||||
httpd.serve_forever()
|
||||
|
||||
return
|
||||
|
||||
|
||||
def main():
|
||||
# Change dir to dir of script
|
||||
abspath = os.path.abspath(__file__)
|
||||
dname = os.path.dirname(abspath)
|
||||
os.chdir(dname)
|
||||
# Arguments:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Serve all directories in a folder over http"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--serve-dir",
|
||||
dest="serve_dir",
|
||||
help="Directory to serve webpages from",
|
||||
type=str,
|
||||
default="./output/final",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--base-port",
|
||||
dest="base_port",
|
||||
help="Initial value of http port",
|
||||
type=int,
|
||||
default=2000,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--increment-number",
|
||||
dest="increment_number",
|
||||
help="Number to increment port num by per webpage",
|
||||
type=int,
|
||||
default=100,
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
dirs = list(
|
||||
filter(
|
||||
lambda item: os.path.isdir(item),
|
||||
map(lambda item: args.serve_dir + "/" + item, os.listdir(args.serve_dir)),
|
||||
)
|
||||
)
|
||||
servers = []
|
||||
for dir in dirs:
|
||||
port = args.base_port + (args.increment_number * dirs.index(dir))
|
||||
print(f"{dir} will be served on http://127.0.0.1:{port}")
|
||||
p = multiprocessing.Process(
|
||||
target=run_webserver,
|
||||
args=(dir, port),
|
||||
)
|
||||
|
||||
servers.append(p)
|
||||
|
||||
for server in servers:
|
||||
server.start()
|
||||
|
||||
for server in servers:
|
||||
server.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
54
shell.nix
54
shell.nix
@ -4,25 +4,49 @@
|
||||
overlays = [ ];
|
||||
},
|
||||
}:
|
||||
let
|
||||
treefmt-nixSrc = builtins.fetchTarball "https://github.com/numtide/treefmt-nix/archive/refs/heads/master.tar.gz";
|
||||
treefmt-nix = import treefmt-nixSrc;
|
||||
in
|
||||
pkgs.mkShell {
|
||||
nativeBuildInputs = with pkgs; [
|
||||
# Needed for standard build
|
||||
coreutils
|
||||
findutils
|
||||
gnused
|
||||
gnugrep
|
||||
gnumake
|
||||
rsync
|
||||
# The main required tool python
|
||||
python3
|
||||
# needed by lxml
|
||||
libxslt
|
||||
libxml2
|
||||
iconv
|
||||
wget
|
||||
# Needed for the site index script
|
||||
python312
|
||||
python312Packages.beautifulsoup4
|
||||
# Needed only for non dev builds. IE --build-env "fsfe.org" or such
|
||||
# For less compilation
|
||||
lessc
|
||||
# Needed for translation status script
|
||||
perl
|
||||
# Needed for git clean in full rebuilds
|
||||
git
|
||||
# Needed for compiling minifiers
|
||||
libffi
|
||||
go
|
||||
# Formatter
|
||||
(treefmt-nix.mkWrapper pkgs {
|
||||
# Used to find the project root
|
||||
projectRootFile = "shell.nix";
|
||||
enableDefaultExcludes = true;
|
||||
programs = {
|
||||
ruff-check.enable = true;
|
||||
ruff-format.enable = true;
|
||||
nixfmt.enable = true;
|
||||
};
|
||||
settings = {
|
||||
global = {
|
||||
on-unmatched = "debug";
|
||||
excludes = [
|
||||
".nltk_data"
|
||||
".venv"
|
||||
];
|
||||
};
|
||||
};
|
||||
})
|
||||
];
|
||||
shellHook = ''
|
||||
export PIP_DISABLE_PIP_VERSION_CHECK=1;
|
||||
python -m venv .venv
|
||||
source .venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
'';
|
||||
}
|
||||
|
||||
@ -1,203 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
exec 2>/dev/null
|
||||
|
||||
DATADIR="data"
|
||||
readonly DATADIR
|
||||
|
||||
if [ "$QUERY_STRING" = "full_build" ]; then
|
||||
if printf %s "$HTTP_REFERER" | grep -qE '^https?://([^/]+\.)?fsfe\.org/'; then
|
||||
touch ./"$DATADIR"/full_build
|
||||
fi
|
||||
printf 'Location: ./\n\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
timestamp() {
|
||||
date -d "@$1" +"%F %T (%Z)"
|
||||
}
|
||||
|
||||
DATADIR="data"
|
||||
readonly DATADIR
|
||||
|
||||
duration() {
|
||||
minutes=$(($1 / 60))
|
||||
if [ "${minutes}" == "1" ]; then
|
||||
minutes="${minutes} minute"
|
||||
else
|
||||
minutes="${minutes} minutes"
|
||||
fi
|
||||
|
||||
seconds=$(($1 % 60))
|
||||
if [ "${seconds}" == "1" ]; then
|
||||
seconds="${seconds} second"
|
||||
else
|
||||
seconds="${seconds} seconds"
|
||||
fi
|
||||
echo "${minutes} ${seconds}"
|
||||
}
|
||||
|
||||
htmlcat() {
|
||||
sed 's;&;\&\;;g;
|
||||
s;<;\<\;;g;
|
||||
s;>;\>\;;g;
|
||||
s;";\"\;;g;
|
||||
s;'\'';\&apos\;;g;' $@
|
||||
}
|
||||
|
||||
start_time=$(cat "$DATADIR/start_time" || stat -c %Y "$0" || echo 0)
|
||||
t_gitupdate=$(stat -c %Y "$DATADIR/GITlatest" || echo 0)
|
||||
t_phase_1=$(stat -c %Y "$DATADIR/phase_1" || echo 0)
|
||||
t_makefile=$(stat -c %Y "$DATADIR/Makefile" || echo 0)
|
||||
t_phase_2=$(stat -c %Y "$DATADIR/phase_2" || echo 0)
|
||||
t_manifest=$(stat -c %Y "$DATADIR/manifest" || echo 0)
|
||||
t_stagesync=$(stat -c %Y "$DATADIR/stagesync" || echo 0)
|
||||
end_time=$(cat "$DATADIR/end_time" || echo 0)
|
||||
duration=$(($end_time - $start_time))
|
||||
term_status=$(if [ "$duration" -gt 0 -a -f "$DATADIR"/lasterror ]; then
|
||||
echo Error
|
||||
elif [ "$duration" -gt 0 ]; then
|
||||
echo Success
|
||||
fi)
|
||||
|
||||
printf %s\\n\\n "Content-Type: text/html;charset=utf-8"
|
||||
sed -e '/<!--\ spacing-comment\ -->/,$d' template.en.html
|
||||
cat <<-HTML_END
|
||||
<h1>Build report</h1>
|
||||
<dl class="buildinfo">
|
||||
<dt>Start time:</dt><dd>$(timestamp ${start_time})</dd>
|
||||
<dt>End time:</dt><dd>$([ "$duration" -gt 0 ] && timestamp ${end_time})</dd>
|
||||
<dt>Duration:</dt><dd>$([ "$duration" -gt 0 ] && duration ${duration})</dd>
|
||||
<dt>Termination Status:</dt><dd>${term_status:-running...}</dd>
|
||||
</dl>
|
||||
$(if [ -f ./$DATADIR/full_build ]; then
|
||||
printf '<span class="fullbuild">Full rebuild will be started within next minute.</span>'
|
||||
else
|
||||
printf '<a class="fullbuild" href="./?full_build">Schedule full rebuild</a>'
|
||||
fi)
|
||||
<details>
|
||||
<summary>Previous builds</summary>
|
||||
<div class="scrollbox">
|
||||
<a href="/${PWD##*/}">latest</a><br/>
|
||||
$(
|
||||
find "$DATADIR" -name "status_*.html" -type f -printf "%f\n" | sort -r | head -n10 | while read stat; do
|
||||
t="${stat#status_}"
|
||||
t="${t%.html}"
|
||||
printf '<a href="%s">%s</a> - %s<br/>' \
|
||||
"/${PWD##*/}/${DATADIR}/$stat" "$(timestamp "$t")" "$(sed -rn 's;^.*<dt>Duration:</dt><dd>(.+)</dd>.*$;\1;p;T;q' "$stat")"
|
||||
printf $'\n'
|
||||
done
|
||||
)
|
||||
</div>
|
||||
</details>
|
||||
<details>
|
||||
<summary>GIT changes:
|
||||
$(
|
||||
if [ ${start_time} -le ${t_gitupdate} ]; then
|
||||
echo "at $(timestamp ${t_gitupdate})" "</summary>" \
|
||||
"<div class=\"scrollbox\">" \
|
||||
"<pre>$(htmlcat "$DATADIR"/GITlatest)</pre>" \
|
||||
"checked"
|
||||
else
|
||||
echo "Unconditional build, changes ignored" \
|
||||
"</summary>" \
|
||||
"<div class=\"scrollbox\">"
|
||||
fi
|
||||
)
|
||||
</div>
|
||||
</details>
|
||||
<details>
|
||||
<summary>Phase 1:
|
||||
$(
|
||||
if [ $start_time -lt $t_phase_1 -a $start_time -lt $t_gitupdate ]; then
|
||||
echo "$(duration $(($t_phase_1 - $t_gitupdate)))" "</summary>" \
|
||||
"<div class=\"scrollbox\">" \
|
||||
"<pre>$(htmlcat "$DATADIR"/phase_1)</pre>"
|
||||
elif [ $start_time -lt $t_phase_1 ]; then
|
||||
echo "$(duration $(($t_phase_1 - $start_time)))" "</summary>" \
|
||||
"<div class=\"scrollbox\">" \
|
||||
"<pre>$(htmlcat "$DATADIR"/phase_1)</pre>"
|
||||
else
|
||||
echo "waiting" "</summary>" \
|
||||
"<div class=\"scrollbox\">"
|
||||
fi
|
||||
)
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details>
|
||||
<summary>Phase 2 Makefile:
|
||||
$(
|
||||
if [ $start_time -lt $t_makefile ]; then
|
||||
echo "$(duration $(($t_makefile - $t_phase_1)))" "</summary>" \
|
||||
"<div class=\"scrollbox\">" \
|
||||
"<pre>$(htmlcat "$DATADIR"/Makefile)</pre>"
|
||||
else
|
||||
echo "waiting" "</summary>" \
|
||||
"<div class=\"scrollbox\">"
|
||||
fi
|
||||
)
|
||||
</div>
|
||||
</details>
|
||||
<details>
|
||||
<summary>Phase 2:
|
||||
$(
|
||||
if [ $start_time -lt $t_phase_2 ]; then
|
||||
echo "$(duration $(($t_phase_2 - $t_makefile)))" "</summary>" \
|
||||
"<div class=\"scrollbox\">" \
|
||||
"<pre>$(htmlcat "$DATADIR"/phase_2)</pre>"
|
||||
else
|
||||
echo "waiting" "</summary>" \
|
||||
"<div class=\"scrollbox\">"
|
||||
fi
|
||||
)
|
||||
</div>
|
||||
</details>
|
||||
<details>
|
||||
<summary>Target update:
|
||||
$(
|
||||
if [ ${start_time} -lt ${t_stagesync} -a -s "$DATADIR"/stagesync ]; then
|
||||
echo "$(($(wc -l "$DATADIR"/stagesync | cut -f1 -d\ ) - 4)) updated files" "</summary>" \
|
||||
"<div class=\"scrollbox\">" \
|
||||
"<pre>$(htmlcat "$DATADIR"/stagesync)</pre>"
|
||||
elif [ -z ${term_status} ]; then
|
||||
echo "waiting" "</summary>" \
|
||||
"<div class=\"scrollbox\">"
|
||||
else
|
||||
echo "-" "</summary>" \
|
||||
"<div class=\"scrollbox\">"
|
||||
fi
|
||||
)
|
||||
</div>
|
||||
</details>
|
||||
<details>
|
||||
<summary>Errors:
|
||||
$(
|
||||
if [ -f lasterror ]; then
|
||||
echo "There were errors" "</summary>" \
|
||||
"<div class=\"scrollbox\">" \
|
||||
"<pre>$(htmlcat "$DATADIR"/lasterror)</pre>" \
|
||||
"checked"
|
||||
else
|
||||
echo "none" "</summary>" \
|
||||
"<div class=\"scrollbox\">"
|
||||
fi
|
||||
)
|
||||
</div>
|
||||
</details>
|
||||
<details>
|
||||
<summary>File Manifest:
|
||||
$(
|
||||
if [ $start_time -lt $t_manifest ]; then
|
||||
echo "$(wc -l < "$DATADIR"/manifest) files" "</summary>" \
|
||||
"<div class=\"scrollbox\">" \
|
||||
"<a href=\"$DATADIR/manifest\">view</a>"
|
||||
else
|
||||
echo "waiting" "</summary>" \
|
||||
"<div class=\"scrollbox\">"
|
||||
fi
|
||||
)
|
||||
</div>
|
||||
</details>
|
||||
HTML_END
|
||||
sed -n -e '/<\/body>/,$p' template.en.html
|
||||
@ -1,10 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<html external="true">
|
||||
<version>0</version>
|
||||
<head>
|
||||
<title>Build Status</title>
|
||||
</head>
|
||||
<body>
|
||||
<!-- spacing-comment -->
|
||||
</body>
|
||||
</html>
|
||||
@ -32,19 +32,16 @@
|
||||
|
||||
<h2>Website status</h2>
|
||||
<p>The following status information is available:</p>
|
||||
<h2>fsfe.org</h2>
|
||||
<h2>Builds</h2>
|
||||
<ul>
|
||||
<li>
|
||||
<a href="fsfe.org/index.cgi">Last build script output</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="translations/">Translation status</a>
|
||||
<a href="https://drone.fsfe.org/FSFE/fsfe-website">Builds of the website</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h2>test.fsfe.org</h2>
|
||||
<h2>Translations</h2>
|
||||
<ul>
|
||||
<li>
|
||||
<a href="test.fsfe.org/index.cgi">Last build script output</a>
|
||||
<a href="translations/">Translation status</a>
|
||||
</li>
|
||||
</ul>
|
||||
</body>
|
||||
|
||||
@ -1 +0,0 @@
|
||||
../fsfe.org/index.cgi
|
||||
@ -1,10 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<html external="true">
|
||||
<version>0</version>
|
||||
<head>
|
||||
<title>Build Status</title>
|
||||
</head>
|
||||
<body>
|
||||
<!-- spacing-comment -->
|
||||
</body>
|
||||
</html>
|
||||
@ -1,25 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Makes all access/modified file dates in the whole repository match with the file's last git commit date
|
||||
# This is important because Make is based on file timestamps, not git commits
|
||||
|
||||
# We have the -z flag so that git does not replace unicode chars with escape codes, and quote the filenames
|
||||
# This also uses null bytes instead of newlines, so we swap them
|
||||
# Only operate on xml and xhtml files. Faster, and all thats needed for translation-status.sh
|
||||
files=$(git ls-files -z "$(git rev-parse --show-toplevel)" | sed 's/\x0/\n/g' | grep -E '^[a-z\.]+\.[a-z]+/.*(xml|xhtml)$')
|
||||
|
||||
total=$(echo "$files" | wc -l)
|
||||
i=1
|
||||
echo "$files" | while read -r file; do
|
||||
echo "[${i}/${total}] $file"
|
||||
# TODO the line directly below this is because after moving main website to fsfe.org dir the translation status
|
||||
# stuff based on dates became a bit useless.
|
||||
# So we use the second to last commit date for every file.
|
||||
# after 6 months or so (february 2025) remove the line below with --follow in it, and uncomment the touch underneath it
|
||||
# TLDR: If after February 2025 remove line directly below this, containign follow and uncomment the touch line below that,
|
||||
# without a follow. Please also remove this comment then
|
||||
touch -a -m --date="@$(git log --pretty="%ct" --follow -2 "$file"| tail -n1)" "$file"
|
||||
# touch -a -m --date="@$(git log --pretty="%ct" -1 "$file")" "$file"
|
||||
((i++))
|
||||
done
|
||||
@ -1,225 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
while getopts h OPT; do
|
||||
case $OPT in
|
||||
h)
|
||||
print_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option: -$OPTARG"
|
||||
print_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
|
||||
readonly SCRIPT_DIR
|
||||
REPO="$SCRIPT_DIR"/../..
|
||||
readonly REPO
|
||||
OUT="${REPO}"/status.fsfe.org/translations/data
|
||||
readonly OUT
|
||||
OUT_TMP="${REPO}"/status.fsfe.org/translations/data-tmp
|
||||
readonly OUT_TMP
|
||||
|
||||
cd "${REPO}" || exit 2
|
||||
|
||||
echo "Making required directories!"
|
||||
mkdir -p "$OUT"
|
||||
mkdir -p "$OUT_TMP"
|
||||
|
||||
LOGFILE="${OUT_TMP}/log.txt"
|
||||
|
||||
langs="$(
|
||||
find ./global/languages -type f -printf "%f\n" | while read -r lang; do
|
||||
echo "$lang $(cat ./global/languages/"$lang")"
|
||||
done
|
||||
)"
|
||||
readonly langs
|
||||
texts_dir="global/data/texts"
|
||||
readonly texts_dir
|
||||
texts_en=$(grep 'id=".*"' ${texts_dir}/texts.en.xml | perl -pe 's/.*id=\"(.*?)\".*/\1/g')
|
||||
readonly texts_en
|
||||
|
||||
# Make filedates match git commits
|
||||
echo "Begin syncing filedates with git commit dates" | tee "${LOGFILE}"
|
||||
"$SCRIPT_DIR"/filedate-sync-git.sh >>"${LOGFILE}"
|
||||
echo "File date sync finished" | tee -a "${LOGFILE}"
|
||||
files=""
|
||||
# Grouped by priority
|
||||
files+=$'\n'$(find ./fsfe.org/index.en.xhtml | sed 's/$/ 1/')
|
||||
files+=$'\n'$(find ./fsfe.org/freesoftware/freesoftware.en.xhtml | sed 's/$/ 1/')
|
||||
|
||||
files+=$'\n'$(find ./fsfe.org/activities -type f \( -iname "activity\.en\.xml" \) | sed 's/$/ 2/')
|
||||
|
||||
files+=$'\n'$(find ./fsfe.org/activities -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 3/')
|
||||
files+=$'\n'$(find ./fsfe.org/freesoftware -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 3/')
|
||||
|
||||
files+=$'\n'$(find ./fsfe.org/order -maxdepth 1 -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 4/')
|
||||
files+=$'\n'$(find ./fsfe.org/contribute -maxdepth 1 -type f \( -iname "spreadtheword*\.en\.xhtml" -o -iname "spreadtheword*\.en\.xml" \) | sed 's/$/ 4/')
|
||||
|
||||
files+=$'\n'$(find ./fsfe.org -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) -mtime -200 -path './fsfe.org/news' -prune -path './fsfe.org/events' -prune | sed 's/$/ 5/')
|
||||
files+=$'\n'$(find ./fsfe.org/news -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) -mtime -30 | sed 's/$/ 5/')
|
||||
# Remove files that are not in the list of those managed by git
|
||||
tmp=""
|
||||
files="$(
|
||||
echo "$files" | while read -r line priority; do
|
||||
if [[ "$(git ls-files | grep -o "${line:2}" | wc -l)" -ge 1 ]] && [[ "$(echo "$tmp" | grep "$line")" == "" ]]; then
|
||||
echo "$line" "$priority"
|
||||
tmp+="$line"
|
||||
fi
|
||||
done
|
||||
)"
|
||||
unset tmp
|
||||
files=$(echo "$files" | grep -v "internal\|order/data/items\.en\.xml\|donate/germany\|donate/netherlands\|donate/switzerland\|status.fsfe.org\|boilerplate\|/\..*\.xml\|)")
|
||||
readonly files
|
||||
|
||||
prevlang=""
|
||||
prevprio=""
|
||||
echo "Begin generating translation status for all languages" | tee -a "$LOGFILE"
|
||||
statuses="$(
|
||||
echo "$files" | while read -r fullname priority; do
|
||||
ext="${fullname##*.}"
|
||||
base="${fullname//\.[a-z][a-z]\.${ext}/}"
|
||||
original_version=$(xsltproc build/xslt/get_version.xsl "$base".en."$ext")
|
||||
echo "$langs" | while read -r lang_short lang_long; do
|
||||
i="$base"."$lang_short"."$ext"
|
||||
echo "Processing file $i" >>"$LOGFILE"
|
||||
if [[ -f $i ]]; then
|
||||
translation_version=$(xsltproc build/xslt/get_version.xsl "$i")
|
||||
else
|
||||
translation_version="-1"
|
||||
fi
|
||||
if [ "${translation_version:-0}" -lt "${original_version:-0}" ]; then
|
||||
# TODO the line directly below this is because after moving main website to fsfe.org dir the translation status
|
||||
# stuff based on dates became a bit useless.
|
||||
# So we use the second to last commit date for every file.
|
||||
# after 6 months or so (february 2025) remove the line below with --follow in it, and uncomment the touch underneath it
|
||||
# TLDR: If after February 2025 remove line directly below this, containign follow and uncomment the touch line below that,
|
||||
# without a follow. Please also remove this comment then
|
||||
originaldate=$(git log --follow --pretty="%ct" -2 "$base".en."$ext" | tail -n1)
|
||||
# originaldate=$(git log --pretty="%ct" -1 "$base".en."$ext")
|
||||
if [ "$ext" == "xhtml" ]; then
|
||||
original_url="https://webpreview.fsfe.org?uri=${base/#\.\/fsfe.org/}.en.html"
|
||||
translation_url="https://webpreview.fsfe.org?uri=${base/#\.\/fsfe.org/}.$lang_short.html"
|
||||
elif [ "$ext" == "xml" ]; then
|
||||
original_url="https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${base/#\.\//}.en.xml"
|
||||
translation_url="https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${base/#\.\//}.$lang_short.xml"
|
||||
else
|
||||
translation_url="#"
|
||||
original_url="#"
|
||||
fi
|
||||
if [ "$translation_version" == "-1" ]; then
|
||||
translation_url="#"
|
||||
fi
|
||||
echo "$lang_short $lang_long $base $priority $originaldate $original_url $original_version $translation_url ${translation_version/-1/Untranslated}"
|
||||
fi
|
||||
done
|
||||
done | sort -t' ' -k 1,1 -k 4,4 -k 3,3 | cat - <(echo "zz zz zz zz zz zz zz zz zz")
|
||||
)"
|
||||
echo "Status Generated" | tee -a "$LOGFILE"
|
||||
|
||||
echo "Generate language status overview" | tee -a "$LOGFILE"
|
||||
cat >"${OUT_TMP}/langs.en.xml" <<-EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<translation-overall-status>
|
||||
<version>1</version>
|
||||
EOF
|
||||
echo "$langs" | while read -r lang_short lang_long; do
|
||||
declare -A prio_counts
|
||||
for i in {1..6}; do
|
||||
prio_counts["$i"]="<priority number=\"$i\" value=\"$(echo "$statuses" | sed -n "/^$lang_short\ $lang_long\ [^\ ]*\ $i/p" | wc -l)\"/>"
|
||||
done
|
||||
cat >>"${OUT_TMP}/langs.en.xml" <<-EOF
|
||||
<language short="$lang_short" long="$lang_long">
|
||||
${prio_counts["1"]}
|
||||
${prio_counts["2"]}
|
||||
</language>
|
||||
EOF
|
||||
unset prio_counts
|
||||
done
|
||||
cat >>"${OUT_TMP}/langs.en.xml" <<-EOF
|
||||
</translation-overall-status>
|
||||
EOF
|
||||
echo "Finished Generating status overview" | tee -a "$LOGFILE"
|
||||
echo "Create language pages" | tee -a "$LOGFILE"
|
||||
echo "$statuses" | while read -r lang_short lang_long page priority originaldate original_url original_version translation_url translation_version; do
|
||||
if [[ "$prevlang" != "$lang_short" ]]; then
|
||||
if [[ "$prevlang" != "" ]]; then
|
||||
# Translatable strings
|
||||
texts_file="${texts_dir}/texts.${prevlang}.xml"
|
||||
missing_texts=()
|
||||
longest_text_length=0
|
||||
for text in $texts_en; do
|
||||
if ! xmllint --xpath "//text[@id = \"${text}\"]" "${texts_file}" &>/dev/null; then
|
||||
missing_texts+=("$text")
|
||||
tmp_length="${#text}"
|
||||
if [ "$tmp_length" -gt "$longest_text_length" ]; then
|
||||
longest_text_length="$tmp_length"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
for index in "${!missing_texts[@]}"; do
|
||||
missing_texts["$index"]="<text>${missing_texts["$index"]}</text>"$'\n'
|
||||
done
|
||||
|
||||
cat >>"${OUT_TMP}/translations.$prevlang.xml" <<-EOF
|
||||
</priority>
|
||||
<missing-texts>
|
||||
<url>https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${texts_file}</url>
|
||||
<max-length>$((longest_text_length + 5))ch</max-length>
|
||||
<filename> $texts_file</filename>
|
||||
${missing_texts[*]}
|
||||
</missing-texts>
|
||||
</translation-status>
|
||||
EOF
|
||||
if [[ "$lang_short" == "zz" ]]; then
|
||||
break
|
||||
fi
|
||||
fi
|
||||
cat >"${OUT_TMP}/translations.$lang_short.xml" <<-EOF
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
||||
<translation-status>
|
||||
<version>1</version>
|
||||
EOF
|
||||
prevprio=""
|
||||
prevlang=$lang_short
|
||||
fi
|
||||
if [[ "$priority" != "$prevprio" ]]; then
|
||||
if [[ "$prevprio" != "" ]]; then
|
||||
cat >>"${OUT_TMP}/translations.$lang_short.xml" <<-EOF
|
||||
</priority>
|
||||
EOF
|
||||
fi
|
||||
cat >>"${OUT_TMP}/translations.$lang_short.xml" <<-EOF
|
||||
<priority value="$priority">
|
||||
EOF
|
||||
prevprio=$priority
|
||||
fi
|
||||
orig=$(date +"%Y-%m-%d" --date="@$originaldate")
|
||||
|
||||
if [[ $originaldate -gt $(date +%s --date='6 months ago') ]]; then
|
||||
new=true
|
||||
else
|
||||
new=false
|
||||
fi
|
||||
cat >>"${OUT_TMP}/translations.$lang_short.xml" <<-EOF
|
||||
<file>
|
||||
<page>$page</page>
|
||||
<original_date>$orig</original_date>
|
||||
<new>$new</new>
|
||||
<original_url>$original_url</original_url>
|
||||
<original_version>$original_version</original_version>
|
||||
<translation_url>$translation_url</translation_url>
|
||||
<translation_version>$translation_version</translation_version>
|
||||
</file>
|
||||
EOF
|
||||
done
|
||||
echo "Finished creating language pages" | tee -a "$LOGFILE"
|
||||
|
||||
rsync -avz --delete --remove-source-files "$OUT_TMP"/ "$OUT"/
|
||||
echo "Finished !"
|
||||
6
status.fsfe.org/translations/__init__.py
Normal file
6
status.fsfe.org/translations/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
@ -36,6 +36,7 @@
|
||||
</table>
|
||||
</details>
|
||||
</xsl:template>
|
||||
|
||||
<xsl:template match="translation-status">
|
||||
<xsl:if test="/buildinfo/@language != 'en'">
|
||||
<div class="translation-status">
|
||||
@ -61,31 +62,31 @@
|
||||
<xsl:for-each select="file">
|
||||
<tr>
|
||||
<td>
|
||||
<xsl:value-of select="page"/>
|
||||
<xsl:value-of select="@page"/>
|
||||
</td>
|
||||
<td>
|
||||
<xsl:value-of select="original_date"/>
|
||||
<xsl:value-of select="@original_date"/>
|
||||
</td>
|
||||
<td>
|
||||
<a>
|
||||
<xsl:attribute name="href">
|
||||
<xsl:value-of select="original_url"/>
|
||||
<xsl:value-of select="@original_url"/>
|
||||
</xsl:attribute>
|
||||
<xsl:value-of select="original_version"/>
|
||||
<xsl:value-of select="@original_version"/>
|
||||
</a>
|
||||
</td>
|
||||
<td>
|
||||
<xsl:choose>
|
||||
<xsl:when test="translation_url != '#'">
|
||||
<xsl:when test="@translation_url != '#'">
|
||||
<a>
|
||||
<xsl:attribute name="href">
|
||||
<xsl:value-of select="translation_url"/>
|
||||
<xsl:value-of select="@translation_url"/>
|
||||
</xsl:attribute>
|
||||
<xsl:value-of select="translation_version"/>
|
||||
<xsl:value-of select="@translation_version"/>
|
||||
</a>
|
||||
</xsl:when>
|
||||
<xsl:otherwise>
|
||||
<xsl:value-of select="translation_version"/>
|
||||
<xsl:value-of select="@translation_version"/>
|
||||
</xsl:otherwise>
|
||||
</xsl:choose>
|
||||
</td>
|
||||
@ -95,26 +96,27 @@
|
||||
</details>
|
||||
</xsl:for-each>
|
||||
<xsl:for-each select="/buildinfo/document/set/missing-texts">
|
||||
<xsl:variable name="max-length" select="max-length"/>
|
||||
<h3>
|
||||
<xsl:text>Missing texts in </xsl:text>
|
||||
<a>
|
||||
<xsl:attribute name="href">
|
||||
<xsl:value-of select="url"/>
|
||||
<xsl:value-of select="@url"/>
|
||||
</xsl:attribute>
|
||||
<xsl:value-of select="filename"/>
|
||||
<xsl:value-of select="@filepath"/>
|
||||
</a>
|
||||
</h3>
|
||||
<h4>
|
||||
<xsl:text>English language texts version: </xsl:text>
|
||||
<xsl:value-of select="@en"/>
|
||||
<br/>
|
||||
<xsl:text>Current language language texts version: </xsl:text>
|
||||
<xsl:value-of select="@curr_lang"/>
|
||||
</h4>
|
||||
<details>
|
||||
<summary>Show</summary>
|
||||
<div style="display: flex; flex-wrap: wrap;">
|
||||
<div style="display: flex; flex-wrap: wrap; align-items: stretch; align-content: stretch; justify-content: space-evenly; gap: 10px;">
|
||||
<xsl:for-each select="text">
|
||||
<div>
|
||||
<xsl:attribute name="style">
|
||||
<xsl:text>width: </xsl:text>
|
||||
<xsl:value-of select="$max-length"/>
|
||||
<xsl:text>;</xsl:text>
|
||||
</xsl:attribute>
|
||||
<xsl:value-of select="current()"/>
|
||||
</div>
|
||||
</xsl:for-each>
|
||||
@ -124,4 +126,5 @@
|
||||
</div>
|
||||
</xsl:if>
|
||||
</xsl:template>
|
||||
|
||||
</xsl:stylesheet>
|
||||
|
||||
256
status.fsfe.org/translations/subdir.py
Normal file
256
status.fsfe.org/translations/subdir.py
Normal file
@ -0,0 +1,256 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
import datetime
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import lxml.etree as etree
|
||||
|
||||
from build.lib.misc import get_basepath, get_version, run_command, update_if_changed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _update_mtime(
|
||||
file: Path,
|
||||
) -> None:
|
||||
logger.debug(f"Updating mtime of {file}")
|
||||
result = run_command(["git", "log", '--pretty="%ct"', "-1", file])
|
||||
time = int(result.strip('"'))
|
||||
os.utime(file, (time, time))
|
||||
|
||||
|
||||
def _generate_translation_data(lang: str, priority: int, file: Path) -> dict:
|
||||
page = get_basepath(file)
|
||||
ext = file.suffix.removeprefix(".")
|
||||
working_file = file.with_suffix("").with_suffix(f".{lang}.{ext}")
|
||||
|
||||
original_date = datetime.datetime.fromtimestamp(file.stat().st_mtime)
|
||||
|
||||
original_version = str(get_version(file))
|
||||
translation_version = (
|
||||
str(get_version(working_file)) if working_file.exists() else "untranslated"
|
||||
)
|
||||
|
||||
original_url = (
|
||||
f"https://webpreview.fsfe.org?uri=/{page.relative_to(page.parts[0])}.en.html"
|
||||
if ext == "xhtml"
|
||||
else (
|
||||
f"https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/{page}.en.xml"
|
||||
if ext == "xml"
|
||||
else "#"
|
||||
)
|
||||
)
|
||||
translation_url = (
|
||||
"#"
|
||||
if not working_file.exists()
|
||||
else (
|
||||
f"https://webpreview.fsfe.org?uri=/{page.relative_to(page.parts[0])}.{lang}.html"
|
||||
if ext == "xhtml"
|
||||
else (
|
||||
f"https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/{page}.{lang}.xml"
|
||||
if ext == "xml"
|
||||
else "#"
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
return (
|
||||
{
|
||||
"page": str(page),
|
||||
"original_date": str(original_date),
|
||||
"original_url": str(original_url),
|
||||
"original_version": str(original_version),
|
||||
"translation_url": str(translation_url),
|
||||
"translation_version": str(translation_version),
|
||||
}
|
||||
if translation_version != original_version
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def _get_text_ids(file: Path) -> list[str]:
|
||||
texts_tree = etree.parse(file)
|
||||
root = texts_tree.getroot()
|
||||
return list(
|
||||
filter(
|
||||
lambda text_id: text_id is not None,
|
||||
map(lambda elem: elem.get("id"), root.iter()),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _create_overview(
|
||||
target_dir: Path,
|
||||
data: dict[str : dict[int : list[dict]]],
|
||||
):
|
||||
work_file = target_dir.joinpath("langs.en.xml")
|
||||
work_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Create the root element
|
||||
page = etree.Element("translation-overall-status")
|
||||
|
||||
# Add the subelements
|
||||
version = etree.SubElement(page, "version")
|
||||
version.text = "1"
|
||||
for lang in data:
|
||||
language_elem = etree.SubElement(
|
||||
page,
|
||||
"language",
|
||||
short=lang,
|
||||
long=Path(f"global/languages/{lang}").read_text().strip(),
|
||||
)
|
||||
for prio in list(data[lang].keys())[:2]:
|
||||
etree.SubElement(
|
||||
language_elem,
|
||||
"priority",
|
||||
number=str(prio),
|
||||
value=str(len(data[lang][prio])),
|
||||
)
|
||||
|
||||
result_str = etree.tostring(page, xml_declaration=True, encoding="utf-8").decode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
update_if_changed(work_file, result_str)
|
||||
|
||||
|
||||
def _create_translation_file(
|
||||
target_dir: Path,
|
||||
lang: str,
|
||||
data: dict[int : list[dict]],
|
||||
) -> None:
|
||||
work_file = target_dir.joinpath(f"translations.{lang}.xml")
|
||||
page = etree.Element("translation-status")
|
||||
version = etree.SubElement(page, "version")
|
||||
version.text = "1"
|
||||
for priority in data:
|
||||
prio = etree.SubElement(page, "priority", value=str(priority))
|
||||
for file_data in data[priority]:
|
||||
etree.SubElement(prio, "file", **file_data)
|
||||
|
||||
en_texts_file = Path("global/data/texts/texts.en.xml")
|
||||
lang_texts_file = Path(f"global/data/texts/texts.{lang}.xml")
|
||||
en_texts = _get_text_ids(en_texts_file)
|
||||
lang_texts = _get_text_ids(lang_texts_file) if lang_texts_file.exists() else []
|
||||
|
||||
missing_texts_elem = etree.SubElement(
|
||||
page,
|
||||
"missing-texts",
|
||||
en=str(get_version(en_texts_file)),
|
||||
curr_lang=(
|
||||
str(get_version(lang_texts_file))
|
||||
if lang_texts_file.exists()
|
||||
else "No texts File!"
|
||||
),
|
||||
url=f"https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/{lang_texts_file}",
|
||||
filepath=str(lang_texts_file),
|
||||
)
|
||||
for missing_text in filter(lambda id: id not in lang_texts, en_texts):
|
||||
text_elem = etree.SubElement(missing_texts_elem, "text")
|
||||
text_elem.text = missing_text
|
||||
|
||||
# Save to XML file
|
||||
result_str = etree.tostring(page, xml_declaration=True, encoding="utf-8").decode(
|
||||
"utf-8"
|
||||
)
|
||||
|
||||
update_if_changed(work_file, result_str)
|
||||
|
||||
|
||||
def run(languages: list[str], processes: int, working_dir: Path) -> None:
|
||||
"""
|
||||
Build translation-status xmls for languages where the translation status has changed. Xmls are placed in target_dir, and only languages are processed.
|
||||
"""
|
||||
target_dir = working_dir.joinpath("data/")
|
||||
logger.debug(f"Building index of status of translations into dir {target_dir}")
|
||||
|
||||
result = run_command(
|
||||
["git", "rev-parse", "--show-toplevel"],
|
||||
)
|
||||
|
||||
# TODO
|
||||
# Run generating all this stuff only if some xhtml|xml files have been changed
|
||||
|
||||
# List files separated by a null bytes
|
||||
result = run_command(
|
||||
["git", "ls-files", "-z", result],
|
||||
)
|
||||
|
||||
with multiprocessing.Pool(processes) as pool:
|
||||
pool.map(
|
||||
_update_mtime,
|
||||
filter(
|
||||
lambda path: path.suffix in [".xhtml", ".xml"],
|
||||
# Split on null bytes, strip and then parse into path
|
||||
map(lambda line: Path(line.strip()), result.split("\x00")),
|
||||
),
|
||||
)
|
||||
|
||||
# Generate our file lists by priority
|
||||
# Super hardcoded unfortunately
|
||||
files_by_prio = dict()
|
||||
files_by_prio[1] = list(Path("fsfe.org/").glob("index.en.xhtml")) + list(
|
||||
Path("fsfe.org/freesoftware/").glob("freesoftware.en.xhtml")
|
||||
)
|
||||
files_by_prio[2] = list(Path("fsfe.org/activities/").glob("*/activity.en.xml"))
|
||||
files_by_prio[3] = (
|
||||
list(Path("fsfe.org/activities/").glob("*.en.xhtml"))
|
||||
+ list(Path("fsfe.org/activities/").glob("*.en.xml"))
|
||||
+ list(Path("fsfe.org/freesoftware/").glob("*.en.xhtml"))
|
||||
+ list(Path("fsfe.org/freesoftware/").glob("*.en.xml"))
|
||||
)
|
||||
files_by_prio[4] = (
|
||||
list(Path("fsfe.org/order/").glob("*.en.xml"))
|
||||
+ list(Path("fsfe.org/order/").glob("*.en.xhtml"))
|
||||
+ list(Path("fsfe.org/contribute/").glob("*.en.xml"))
|
||||
+ list(Path("fsfe.org/contribute/").glob("*.en.xhtml"))
|
||||
)
|
||||
files_by_prio[5] = list(Path("fsfe.org/order/").glob("**/*.en.xml")) + list(
|
||||
Path("fsfe.org/order/").glob("**/*.en.xhtml")
|
||||
)
|
||||
|
||||
for priority in sorted(files_by_prio.keys(), reverse=True):
|
||||
files_by_prio[priority] = list(
|
||||
filter(
|
||||
lambda path: not any(
|
||||
[(path in files_by_prio[prio]) for prio in range(1, priority)]
|
||||
),
|
||||
files_by_prio[priority],
|
||||
)
|
||||
)
|
||||
|
||||
files_by_lang_by_prio = {}
|
||||
for lang in languages:
|
||||
files_by_lang_by_prio[lang] = {}
|
||||
for priority in files_by_prio:
|
||||
files_by_lang_by_prio[lang][priority] = list(
|
||||
filter(
|
||||
lambda result: result is not None,
|
||||
pool.starmap(
|
||||
_generate_translation_data,
|
||||
[
|
||||
(lang, priority, file)
|
||||
for file in files_by_prio[priority]
|
||||
],
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
# sadly single treaded, as only one file being operated on
|
||||
_create_overview(target_dir, files_by_lang_by_prio)
|
||||
|
||||
for data in [
|
||||
(target_dir, lang, files_by_lang_by_prio[lang])
|
||||
for lang in files_by_lang_by_prio
|
||||
]:
|
||||
pool.starmap(
|
||||
_create_translation_file,
|
||||
[
|
||||
(target_dir, lang, files_by_lang_by_prio[lang])
|
||||
for lang in files_by_lang_by_prio
|
||||
],
|
||||
)
|
||||
6
tools/__init__.py
Normal file
6
tools/__init__.py
Normal file
@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Free Software Foundation Europe e.V. <https://fsfe.org>
|
||||
#
|
||||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
# __init__.py is a special Python file that allows a directory to become
|
||||
# a Python package so it can be accessed using the 'import' statement.
|
||||
File diff suppressed because one or more lines are too long
@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update default sytlesheets
|
||||
# -----------------------------------------------------------------------------
|
||||
# This script is called from the phase 1 Makefile and places a .default.xsl
|
||||
# into each directory containing source files for HTML pages (*.xhtml). These
|
||||
# .default.xsl are symlinks to the first available actual default.xsl found
|
||||
# when climbing the directory tree upwards, it's the XSL stylesheet to be used
|
||||
# for building the HTML files from this directory.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
set -e
|
||||
|
||||
echo "* Updating default stylesheets"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Get a list of all directories containing .xhtml source files
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
directories=$(
|
||||
find -name "*.??.xhtml" \
|
||||
| xargs dirname \
|
||||
| sort --unique
|
||||
)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# In each dir, place a .default.xsl symlink pointing to the nearest default.xsl
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
for directory in ${directories}; do
|
||||
dir="${directory}"
|
||||
prefix=""
|
||||
until [ -f "${dir}/default.xsl" -o "${dir}" = "." ]; do
|
||||
dir="$(dirname "${dir}")"
|
||||
prefix="${prefix}../"
|
||||
done
|
||||
ln -sf "${prefix}default.xsl" "${directory}/.default.xsl"
|
||||
done
|
||||
@ -1,80 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update local menus
|
||||
# -----------------------------------------------------------------------------
|
||||
# This script is called from the phase 1 Makefile and updates all the
|
||||
# .localmenu.*.xml files containing the local menus.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
set -e
|
||||
|
||||
echo "* Updating local menus"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Get a list of all source files containing local menus
|
||||
# -----------------------------------------------------------------------------
|
||||
all_files=$(
|
||||
find . -name "*.xhtml" -not -name "*-template.*" \
|
||||
| xargs grep -l "</localmenu>" \
|
||||
| sort
|
||||
)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Split that list by localmenu directory
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
declare -A files_by_dir
|
||||
|
||||
for file in ${all_files}; do
|
||||
dir=$(xsltproc build/xslt/get_localmenu_dir.xsl ${file})
|
||||
dir=${dir:-$(dirname ${file})}
|
||||
files_by_dir[${dir}]="${files_by_dir[${dir}]} ${file}"
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# If any of the source files has been updated, rebuild all .localmenu.*.xml
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
for dir in ${!files_by_dir[@]}; do
|
||||
for file in ${files_by_dir[${dir}]}; do
|
||||
if [ "${file}" -nt "${dir}/.localmenu.en.xml" ]; then
|
||||
|
||||
# Find out which languages to generate.
|
||||
languages="$1"
|
||||
|
||||
# Compile the list of base filenames of the source files.
|
||||
basefiles=$(
|
||||
ls ${files_by_dir[${dir}]} \
|
||||
| sed 's/\...\.xhtml//' \
|
||||
| sort --uniq
|
||||
)
|
||||
|
||||
# For each language, create the .localmenu.${lang}.xml file.
|
||||
for lang in $languages; do
|
||||
echo "* Creating ${dir}/.localmenu.${lang}.xml"
|
||||
{
|
||||
echo "<?xml version=\"1.0\"?>"
|
||||
echo ""
|
||||
echo "<feed>"
|
||||
for basefile in ${basefiles}; do
|
||||
if [ -f "${basefile}.${lang}.xhtml" ]; then
|
||||
file="${basefile}.${lang}.xhtml"
|
||||
else
|
||||
file="${basefile}.en.xhtml"
|
||||
fi
|
||||
xsltproc \
|
||||
--stringparam "link" "$(echo "$basefile"| sed 's/^\.\/[^\/]*//').html" \
|
||||
build/xslt/get_localmenu_line.xsl \
|
||||
"${file}"
|
||||
echo ""
|
||||
done | sort
|
||||
echo "</feed>"
|
||||
} > "${dir}/.localmenu.${lang}.xml"
|
||||
done
|
||||
|
||||
# The local menu for this directory has been built, no need to check
|
||||
# further source files.
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
@ -1,56 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update XSL stylesheets (*.xsl) according to their dependency
|
||||
# -----------------------------------------------------------------------------
|
||||
# This script is called from the phase 1 Makefile and updates (actually: just
|
||||
# touches) all XSL files which depend on another XSL file that has changed
|
||||
# since the last build run. The phase 2 Makefile then only has to consider the
|
||||
# directly used stylesheet as a prerequisite for building each file and doesn't
|
||||
# have to worry about other stylesheets imported into that one.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
echo "* Updating XSL stylesheets"
|
||||
|
||||
pid=$$
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Generate a temporary makefile with dependencies for all .xsl files
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
makefile="/tmp/makefile-${pid}"
|
||||
|
||||
{
|
||||
for xsl_file in $(find * -name '*.xsl' -not -name '.default.xsl'); do
|
||||
prerequisites=$(echo $(
|
||||
cat "${xsl_file}" \
|
||||
| tr '\t\r\n' ' ' \
|
||||
| sed -r 's/(<xsl:(include|import)[^>]*>)/\n\1\n/g' \
|
||||
| sed -rn '/<xsl:(include|import)[^>]*>/s/^.*href *= *"([^"]*)".*$/\1/gp' \
|
||||
| xargs -I'{}' realpath --relative-to="." "$(dirname ${xsl_file})/{}" \
|
||||
))
|
||||
if [ -n "${prerequisites}" ]; then
|
||||
echo "all: ${xsl_file}"
|
||||
echo "${xsl_file}: ${prerequisites}"
|
||||
echo ""
|
||||
fi
|
||||
done
|
||||
|
||||
echo '%.xsl:'
|
||||
echo -e '\techo "* Touching $@"'
|
||||
echo -e '\ttouch $@'
|
||||
} > "${makefile}"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Execute the temporary makefile
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
make --silent --file="${makefile}"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Clean up
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
rm -f "${makefile}"
|
||||
@ -1,258 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update XML filelists (*.xmllist) and tag list pages (fsfe.org/tags/tagged-*)
|
||||
# -----------------------------------------------------------------------------
|
||||
# This script is called from the phase 1 Makefile and creates/updates the
|
||||
# following files:
|
||||
#
|
||||
# * fsfe.org/tags/tagged-<tags>.en.xhtml for each tag used. Apart from being
|
||||
# automatically created, these are regular source files for HTML pages, and
|
||||
# in phase 2 are built into pages listing all news items and events for a
|
||||
# tag.
|
||||
#
|
||||
# * fsfe.org/tags/.tags.??.xml with a list of the tags used.
|
||||
#
|
||||
# * <dir>/.<base>.xmllist for each <dir>/<base>.sources as well as for each
|
||||
# fsfe.org/tags/tagged-<tags>.en.xhtml. These files are used in phase 2 to include the
|
||||
# correct XML files when generating the HTML pages. It is taken care that
|
||||
# these files are only updated whenever their content actually changes, so
|
||||
# they can serve as a prerequisite in the phase 2 Makefile.
|
||||
#
|
||||
# Changing or removing tags in XML files is also considered, in which case a
|
||||
# file is removed from the .xmllist files.
|
||||
#
|
||||
# When a tag has been removed from the last XML file where it has been used,
|
||||
# the tagged-* are correctly deleted.
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
pid=$$
|
||||
languages="$1"
|
||||
|
||||
nextyear=$(date --date="next year" +%Y)
|
||||
thisyear=$(date --date="this year" +%Y)
|
||||
lastyear=$(date --date="last year" +%Y)
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Make sure temporary directory is empty
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
tagmaps="/tmp/tagmaps-${pid}"
|
||||
|
||||
rm -rf "${tagmaps}"
|
||||
mkdir "${tagmaps}"
|
||||
|
||||
taglabels="/tmp/taglabels-${pid}"
|
||||
|
||||
rm -rf "${taglabels}"
|
||||
mkdir "${taglabels}"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create a complete and current map of which tag is used in which files
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
echo "* Generating tag maps"
|
||||
|
||||
for xml_file in $(find * -regex "[a-z.]+.[a-z]+/.*/*.\(${languages// /\\\|}\)\.xml" -not -path 'fsfe.org/tags/*' | xargs grep -l '<tag' | sort); do
|
||||
xsltproc "build/xslt/get_tags.xsl" "${xml_file}" | while read tag label; do
|
||||
# Add file to list of files by tag key
|
||||
echo "${xml_file%.??.xml}" >> "${tagmaps}/${tag}"
|
||||
|
||||
# Store label by language and tag key
|
||||
xml_base=${xml_file%.xml}
|
||||
language=${xml_base##*.}
|
||||
if [ "${language}" -a "${label}" ]; then
|
||||
mkdir -p "${taglabels}/${language}"
|
||||
# Always overwrite so the newest news item will win.
|
||||
echo "${label}" > "${taglabels}/${language}/${tag}"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Make sure that each file only appears once per tag map
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
for tag in $(ls "${tagmaps}"); do
|
||||
sort -u "${tagmaps}/${tag}" > "${tagmaps}/tmp"
|
||||
mv "${tagmaps}/tmp" "${tagmaps}/${tag}"
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update only those files where a change has happened
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
for tag in $(ls "${tagmaps}"); do
|
||||
if ! cmp --quiet "${tagmaps}/${tag}" "fsfe.org/tags/.tagged-${tag}.xmllist"; then
|
||||
echo "* Updating tag ${tag}"
|
||||
sed "s,XXX_TAGNAME_XXX,${tag},g" "fsfe.org/tags/tagged.en.xhtml" \
|
||||
> "fsfe.org/tags/tagged-${tag}.en.xhtml"
|
||||
cp "${tagmaps}/${tag}" "fsfe.org/tags/.tagged-${tag}.xmllist"
|
||||
fi
|
||||
done
|
||||
|
||||
rm -f "fsfe.org/tags/tagged-front-page.en.xhtml" # We don't want that one
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Remove the files for tags which have been completely deleted
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
for tag in $(ls "fsfe.org/tags" | sed -rn 's/tagged-(.*)\.en.xhtml/\1/p'); do
|
||||
if [ ! -f "${tagmaps}/${tag}" ]; then
|
||||
echo "* Deleting fsfe.org/tags/tagged-${tag}.en.xhtml"
|
||||
rm "fsfe.org/tags/tagged-${tag}.en.xhtml"
|
||||
fi
|
||||
done
|
||||
|
||||
for tag in $(ls -a "fsfe.org/tags" | sed -rn 's/.tagged-(.*)\.xmllist/\1/p'); do
|
||||
if [ ! -f "${tagmaps}/${tag}" ]; then
|
||||
echo "* Deleting fsfe.org/tags/.tagged-${tag}.xmllist"
|
||||
rm "fsfe.org/tags/.tagged-${tag}.xmllist"
|
||||
fi
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update the tag lists
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
echo "* Updating tag lists"
|
||||
|
||||
taglist="/tmp/taglist-${pid}"
|
||||
|
||||
declare -A filecount
|
||||
|
||||
for section in "news" "events"; do
|
||||
for tag in $(ls "${tagmaps}"); do
|
||||
filecount["${tag}:${section}"]=$(grep "^fsfe.org/${section}/" "${tagmaps}/${tag}" | wc --lines || true)
|
||||
done
|
||||
done
|
||||
|
||||
for language in $(ls ${taglabels}); do
|
||||
{
|
||||
echo '<?xml version="1.0" encoding="UTF-8"?>'
|
||||
echo ''
|
||||
echo '<tagset>'
|
||||
|
||||
for section in "news" "events"; do
|
||||
for tag in $(ls "${tagmaps}"); do
|
||||
if [ "${tag}" == "front-page" ]; then
|
||||
continue # We don't want an actual page for that
|
||||
fi
|
||||
|
||||
count=${filecount["${tag}:${section}"]}
|
||||
|
||||
if [ -f "${taglabels}/${language}/${tag}" ]; then
|
||||
label="$(cat "${taglabels}/${language}/${tag}")"
|
||||
elif [ -f "${taglabels}/en/${tag}" ]; then
|
||||
label="$(cat "${taglabels}/en/${tag}")"
|
||||
else
|
||||
label="${tag}"
|
||||
fi
|
||||
|
||||
if [ "${count}" != "0" ]; then
|
||||
echo " <tag section=\"${section}\" key=\"${tag}\" count=\"${count}\">${label}</tag>"
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
echo '</tagset>'
|
||||
} > ${taglist}
|
||||
|
||||
if ! cmp --quiet "${taglist}" "fsfe.org/tags/.tags.${language}.xml"; then
|
||||
echo "* Updating fsfe.org/tags/.tags.${language}.xml"
|
||||
cp "${taglist}" "fsfe.org/tags/.tags.${language}.xml"
|
||||
fi
|
||||
|
||||
rm -f "${taglist}"
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Remove the temporary directory
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
rm -rf "${tagmaps}"
|
||||
rm -rf "${taglabels}"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update .xmllist files for .sources and .xhtml containing <module>s
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
echo "* Updating XML lists"
|
||||
|
||||
all_xml="$(find * -name '*.??.xml' | sed -r 's/\...\.xml$//' | sort -u)"
|
||||
|
||||
source_bases="$(find * -name "*.sources" | sed -r 's/\.sources$//')"
|
||||
module_bases="$(find * -name "*.??.xhtml" | xargs grep -l '<module' | sed -r 's/\...\.xhtml$//')"
|
||||
all_bases="$((echo "${source_bases}"; echo "${module_bases}") | sort -u)"
|
||||
|
||||
for base in ${all_bases}; do
|
||||
{
|
||||
# Data files defined in the .sources list
|
||||
if [ -f "${base}.sources" ]; then
|
||||
cat "${base}.sources" | while read line; do
|
||||
# Get the pattern from the pattern:[tag] construction
|
||||
pattern=$(echo "${line}" | sed -rn 's/(.*):\[.*\]$/\1/p')
|
||||
|
||||
if [ -z "${pattern}" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# Honor $nextyear, $thisyear, and $lastyear variables
|
||||
pattern=${pattern//\$nextyear/${nextyear}}
|
||||
pattern=${pattern//\$thisyear/${thisyear}}
|
||||
pattern=${pattern//\$lastyear/${lastyear}}
|
||||
|
||||
# Change from a glob pattern into a regex
|
||||
pattern=$(echo "${pattern}" | sed -r -e 's/([.^$[])/\\\1/g; s/\*/.*/g')
|
||||
|
||||
# Get the tag from the pattern:[tag] construction
|
||||
tag=$(echo "${line}" | sed -rn 's/.*:\[(.*)\]$/\1/p')
|
||||
|
||||
# We append || true so the script doesn't fail if grep finds nothing at all
|
||||
if [ -n "${tag}" ]; then
|
||||
cat "fsfe.org/tags/.tagged-${tag}.xmllist"
|
||||
else
|
||||
echo "${all_xml}"
|
||||
fi | grep "^${pattern}\$" || true
|
||||
done
|
||||
fi
|
||||
|
||||
# Data files required for the <module>s used
|
||||
for xhtml_file in ${base}.??.xhtml; do
|
||||
xsltproc "build/xslt/get_modules.xsl" "${xhtml_file}"
|
||||
done
|
||||
} | sort -u > "/tmp/xmllist-${pid}"
|
||||
|
||||
list_file="$(dirname ${base})/.$(basename ${base}).xmllist"
|
||||
|
||||
if ! cmp --quiet "/tmp/xmllist-${pid}" "${list_file}"; then
|
||||
echo "* Updating ${list_file}"
|
||||
cp "/tmp/xmllist-${pid}" "${list_file}"
|
||||
fi
|
||||
|
||||
rm -f "/tmp/xmllist-${pid}"
|
||||
done
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Touch all .xmllist files where one of the contained files has changed
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
echo "* Checking contents of XML lists"
|
||||
|
||||
for list_file in $(find -name '.*.xmllist' -printf '%P\n' | sort); do
|
||||
if [ ! -s "${list_file}" ]; then # Empty file list
|
||||
continue
|
||||
fi
|
||||
|
||||
xml_files="$(sed -r 's/(^.*$)/\1.??.xml/' "${list_file}")"
|
||||
# For long file lists, the following would fail with an exit code of 141
|
||||
# (SIGPIPE), so we must add a "|| true"
|
||||
newest_xml_file="$(ls -t ${xml_files} | head --lines=1 || true)"
|
||||
|
||||
if [ "${newest_xml_file}" -nt "${list_file}" ]; then
|
||||
echo "* Touching ${list_file}"
|
||||
touch "${list_file}"
|
||||
fi
|
||||
done
|
||||
Loading…
x
Reference in New Issue
Block a user