From bb45fc85ba29e1ce680dae3741ebd4fd42f5cda6 Mon Sep 17 00:00:00 2001 From: delliott Date: Tue, 29 Oct 2024 14:56:41 +0000 Subject: [PATCH] feat/build-improvements-2 (#4553) Another attempt at https://git.fsfe.org/FSFE/fsfe-website/pulls/4516 Adds a small fix for tag map generation https://git.fsfe.org/FSFE/fsfe-website/pulls/4516 introduces build failures, but testing indicates that those failures would have been resolved by a full rebuild. Hence this pr. Co-authored-by: Darragh Elliott Reviewed-on: https://git.fsfe.org/FSFE/fsfe-website/pulls/4553 Co-authored-by: delliott Co-committed-by: delliott --- Makefile | 13 ++- README.md | 2 +- build.sh | 51 +++++++--- build/HELP | 58 ----------- build/README.md | 42 ++++++++ build/arguments.sh | 167 ++++++++++++++++---------------- build/build_main.sh | 88 ++++++++--------- build/buildrun.sh | 193 +++++++++++++++++++------------------ build/filenames.sh | 16 +-- build/logging.sh | 50 +++++----- build/makerules.sh | 53 +++++----- build/misc.sh | 26 ++--- build/process_file.sh | 4 +- build/processor.sh | 70 +++++++------- build/scaffold.sh | 190 ++++++++++++++++++------------------ fsfe.org/events/Makefile | 2 +- fsfe.org/news/Makefile | 4 +- tools/update_localmenus.sh | 6 +- tools/update_xmllists.sh | 3 +- 19 files changed, 523 insertions(+), 515 deletions(-) delete mode 100644 build/HELP create mode 100644 build/README.md diff --git a/Makefile b/Makefile index 7f23a0fcab..cb1e59fa80 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ # This will be overwritten in the command line running this Makefile. build_env = development +languages = none # ----------------------------------------------------------------------------- # Build search index @@ -68,7 +69,7 @@ SUBDIRS := $(shell find . -regex "./[a-z\.]+\.[a-z]+/.*/Makefile" | xargs dirnam all: $(SUBDIRS) $(SUBDIRS): .FORCE echo "* Preparing subdirectory $@" - $(MAKE) --silent --directory=$@ + $(MAKE) --silent --directory=$@ languages="$(languages)" # ----------------------------------------------------------------------------- # Create XML symlinks @@ -82,9 +83,7 @@ $(SUBDIRS): .FORCE # otherwise. This symlinks make sure that phase 2 can easily use the right file # for each language, also as a prerequisite in the Makefile. -LANGUAGES := $(shell ls -xw0 global/languages) - -TEXTS_LINKS := $(foreach lang,$(LANGUAGES),global/data/texts/.texts.$(lang).xml) +TEXTS_LINKS := $(foreach lang,$(languages),global/data/texts/.texts.$(lang).xml) all: $(TEXTS_LINKS) global/data/texts/.texts.%.xml: .FORCE @@ -94,7 +93,7 @@ global/data/texts/.texts.%.xml: .FORCE ln -sf texts.en.xml $@; \ fi -TOPBANNER_LINKS := $(foreach lang,$(LANGUAGES),global/data/topbanner/.topbanner.$(lang).xml) +TOPBANNER_LINKS := $(foreach lang,$(languages),global/data/topbanner/.topbanner.$(lang).xml) all: $(TOPBANNER_LINKS) global/data/topbanner/.topbanner.%.xml: .FORCE @@ -134,7 +133,7 @@ default_xsl: .PHONY: localmenus all: localmenus localmenus: $(SUBDIRS) - tools/update_localmenus.sh + tools/update_localmenus.sh "$(languages)" # ----------------------------------------------------------------------------- # Update XML filelists @@ -155,4 +154,4 @@ localmenus: $(SUBDIRS) .PHONY: xmllists all: xmllists xmllists: $(SUBDIRS) - tools/update_xmllists.sh + tools/update_xmllists.sh "$(languages)" diff --git a/README.md b/README.md index 535fd9ee8f..c3c143e042 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ You can see the current status of translation progress of fsfe.org at [status.fs ## Build -There are two ways to build and develop the directory locally. Initial builds of the webpages may take ~40 minutes, but subsequent builds should be much faster. +There are two ways to build and develop the directory locally. Initial builds of the webpages may take ~40 minutes, but subsequent builds should be much faster. Using the `--languages` flag to avoid building all supported languages can make this much faster. See ./build/README.md for more information. Alterations to build scripts or the files used site-wide will result in near full rebuilds. diff --git a/build.sh b/build.sh index 35f8a45294..e201d3afe0 100755 --- a/build.sh +++ b/build.sh @@ -2,26 +2,47 @@ set -euo pipefail usage() { - echo "Usage: $0 -f|--full -s|--serve" 1>&2 - exit 1 + cat <<-EOF + # build.sh Usage + ## General + This script is a wrapper script over ./build/build_main.sh that provides nicer option names, and the options to serve the files. + For documentation on the build script itself see ./build/README.md + ## Flags + ### -f | --full + Perform a full rebuild of the webpages. + ### -s | --serve + Serve the build webpages over localhost. + ### -- + Everything after this is passed directly to build_main. + See ./build/README.md for valid options. + EOF + exit 1 } command="build_run" serve="" +extra_args="" while [ "$#" -gt 0 ]; do - case "$1" in - --full | -f) - command="build_into" && shift 1 - ;; - --serve | -s) - serve="true" && shift 1 - ;; - *) - usage - ;; - esac + case "$1" in + --full | -f) + command="build_into" && shift 1 + ;; + --serve | -s) + serve="true" && shift 1 + ;; + --) + shift 1 + while [ "$#" -gt 0 ]; do + extra_args+="$1 " + shift 1 + done + ;; + *) + usage + ;; + esac done mkdir -p ./output -./build/build_main.sh "$command" ./output/final --statusdir ./output/final/status.fsfe.org/fsfe.org/data +./build/build_main.sh "$command" ./output/final --statusdir ./output/final/status.fsfe.org/fsfe.org/data $extra_args if [[ "$serve" ]]; then - python3 ./serve-websites.py + python3 ./serve-websites.py fi diff --git a/build/HELP b/build/HELP deleted file mode 100644 index 4464d00b7d..0000000000 --- a/build/HELP +++ /dev/null @@ -1,58 +0,0 @@ - -Usage: ------------------------------------------------------------------------------- - -build_main.sh [options] build_into "destination_dir" - Perform the page build. Write output to destination_dir. The source - directory is determined from the build scripts own location. - -build_main.sh [options] build_xmlstream "file.xhtml" - Compile an xml stream from the specified file, additional sources will be - determined and included automatically. The stream is suitable for being - passed into xsltproc. - -build_main.sh [options] process_file "file.xhtml" [processor.xsl] - Generate output from an xhtml file as if it would be processed during the - build. Output is written to STDOUT and can be redirected as desired. - If a xslt file is not given, it will be chosen automatically. - -build_main.sh [options] tree_maker [input_dir] "destination_dir" - Generate a set of make rules to build the website contained in input_dir. - destination_dir should be the www root of a web server. - If input_dir is omitted, it will be the source directory determined from - the build scripts location. - Note: if destination_dir is set via previous options, and only one parameter - is given, then this parameter will be interpreted as input_dir - -OPTIONS -------- - ---source "source_dir" - Force a specific source directory. If not explicitly given source_dir is - determined from the build scripts own location. - Paths given in .sources files are interpreted as relative to source_dir - making this option useful when building a webpage outside of the build - scripts "regular" tree. - ---destination "destination_dir" - The directory into which the website will be built. This option can be used - in conjunction with the tree_maker and build_into commands. It will override - the destination_dir option given after those commands and is therefore - redundant. The option exists to provide backward compatibility to the 2002 - build script. - ---statusdir "status_dir" - A directory to which messages are written. If no status_dir is provided - information will be written to stdout. The directory will also be used - to store some temporary files, which would otherwise be set up in the - system wide temp directory. - ---build-env "selection" - Indicate the current build environment. "selection" can be one of: - * "fsfe.org": building https://fsfe.org on the production server - * "test.fsfe.org": building https://test.fsfe.org on the production server - * "development" (default): local development build - In a local development build, code to dynamically compile the less files into - CSS will be included in the HTML output, while in the other environments, the - precompiles fsfe.min.css (or valentine.min.css) will be referenced from the - generated web pages. diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000000..11a3d0bfde --- /dev/null +++ b/build/README.md @@ -0,0 +1,42 @@ +# Main Commands +Note that targets takes a comma separated list of valid rsync targets, and hence supports ssh targets. If targeting more than one directory one must use the --stage-dir flag documented below. +## build_main.sh [options] build_run "targets" + Perform the page build. Write output to targets. The source directory is determined from the build scripts own location. + +## build_main.sh [options] git_build_into "targets" + Update repo to latest version of upstream branch and then perform a standard build. Write output to targets. The source directory is determined from the build scripts own location. + +## build_main.sh [options] build_into "targets" + Perform a full rebuild of the webpages, removing all cached files. Write output to targets. The source directory is determined from the build scripts own location. + +# Internal Commands +It is unlikely that you will need to directly call these commands, but they are documented here never the less. +## build_main.sh [options] build_xmlstream "file.xhtml" + Compile an xml stream from the specified file, additional sources will be determined and included automatically. The stream is suitable for being passed into xsltproc. + +## build_main.sh [options] process_file "file.xhtml" [processor.xsl] + Generate output from an xhtml file as if it would be processed during the + build. Output is written to STDOUT and can be redirected as desired. + If a xslt file is not given, it will be chosen automatically. + +## build_main.sh [options] tree_maker [input_dir] "targets" + Generate a set of make rules to build the website contained in input_dir. targets should be the www root of a web server. If input_dir is omitted, it will be the source directory determined from the build scripts location. Note: if targets is set via previous options, and only one parameter is given, then this parameter will be interpreted as input_dir + +# OPTIONS +## --source "source_dir" + Force a specific source directory. If not explicitly given source_dir is determined from the build scripts own location. Paths given in .sources files are interpreted as relative to source_dir making this option useful when building a webpage outside of the build scripts "regular" tree. + +## --status-dir "status_dir" +A directory to which messages are written. If no status_dir is provided information will be written to stdout. The directory will also be used to store some temporary files, which would otherwise be set up in the system wide temp directory. + +## --stage-dir "stage_dir" +Directory used for staging the website builds. The websites are first build into this directory, then copied to each targets. + +## --build-env "selection" +Indicate the current build environment. "selection" can be one of: * "fsfe.org": building https://fsfe.org on the production server * "test.fsfe.org": building https://test.fsfe.org on the production server * "development" (default): local development build In a local development build, code to dynamically compile the less files into CSS will be included in the HTML output, while in the other environments, the precompiles fsfe.min.css (or valentine.min.css) will be referenced from the generated web pages. + +## --languages "languages" +Takes a comma separated list of language shot codes to build the website in. For example, to build the site in English and French only one would use `--languages en,fr`. One of the built languages must be English. + +## --help +Show this README. diff --git a/build/arguments.sh b/build/arguments.sh index b05931bb4a..396f7cacd0 100755 --- a/build/arguments.sh +++ b/build/arguments.sh @@ -3,87 +3,92 @@ [ -z "$inc_misc" ] && . "$basedir/build/misc.sh" if [ -z "$inc_arguments" ]; then - inc_arguments=true - basedir="$(realpath "${0%/*}/..")" + inc_arguments=true + basedir="$(realpath "${0%/*}/..")" - while [ "$#" -gt 0 ]; do - case "$1" in - -s|--statusdir|--status-dir) - [ "$#" -gt 0 ] && shift 1 && statusdir="$1" - ;; - --domain) - [ "$#" -gt 0 ] && shift 1 && domain="$1" - ;; - --source) - [ "$#" -gt 0 ] && shift 1 && basedir="$1" - ;; - --stage|--stagedir) - [ "$#" -gt 0 ] && shift 1 && stagedir="$1" - ;; - -d|--dest|--destination) - [ "$#" -gt 0 ] && shift 1 && target="$1" - ;; - --build-env) - [ "$#" -gt 0 ] && shift 1 && build_env="$1" - ;; - -h|--help) - command="help" - ;; - build_into) - command="$1$command" - [ "$#" -gt 0 ] && shift 1 && target="$1" - ;; - git_build_into) - command="$1$command" - [ "$#" -gt 0 ] && shift 1 && target="$1" - ;; - build_run) - command="$1$command" - [ "$#" -gt 0 ] && shift 1 && target="$1" - ;; - build_xmlstream) - command="$1$command" - [ "$#" -gt 0 ] && shift 1 && workfile="$1" - ;; - tree_maker) - command="$1$command" - [ -n "$target" -o -n "$3" ] && shift 1 && tree="$1" - shift 1; [ -n "$1" ] && target="$1" - ;; - process_file) - command="$1$command" - [ "$#" -gt 0 ] && shift 1 && workfile="$1" - [ "$#" -gt 0 ] && shift 1 && processor="$1" - ;; - *) - print_error "Unknown option: $1" - exit 1 - ;; - esac - [ "$#" -gt 0 ] && shift 1 - done - - tree="${tree:-$basedir}" - stagedir="${stagedir:-$target}" - readonly tree="${tree:+$(realpath "$tree")}" - readonly stagedir="${stagedir:+$(realpath "$stagedir")}" - readonly basedir="${basedir:+$(realpath "$basedir")}" - readonly domain="${domain:-www.fsfe.org}" - readonly command - if [ "$stagedir" != "$target" ] && printf %s "$target" | egrep -q '^.+@.+:(.+)?$'; then - readonly target - else - readonly target="${target:+$(realpath "$target")}" - fi + while [ "$#" -gt 0 ]; do + case "$1" in + -s | --statusdir | --status-dir) + [ "$#" -gt 0 ] && shift 1 && statusdir="$1" + ;; + --source) + [ "$#" -gt 0 ] && shift 1 && basedir="$1" + ;; + --stage | --stagedir | --stage-dir) + [ "$#" -gt 0 ] && shift 1 && stagedir="$1" + ;; + --build-env) + [ "$#" -gt 0 ] && shift 1 && build_env="$1" + ;; + --languages) + [ "$#" -gt 0 ] && shift 1 && languages="$1" + ;; + -h | --help) + command="help" + ;; + build_into) + command="$1$command" + [ "$#" -gt 0 ] && shift 1 && target="$1" + ;; + git_build_into) + command="$1$command" + [ "$#" -gt 0 ] && shift 1 && target="$1" + ;; + build_run) + command="$1$command" + [ "$#" -gt 0 ] && shift 1 && target="$1" + ;; + build_xmlstream) + command="$1$command" + [ "$#" -gt 0 ] && shift 1 && workfile="$1" + ;; + tree_maker) + command="$1$command" + [ -n "$target" -o -n "$3" ] && shift 1 && tree="$1" + shift 1 + [ -n "$1" ] && target="$1" + ;; + process_file) + command="$1$command" + [ "$#" -gt 0 ] && shift 1 && workfile="$1" + [ "$#" -gt 0 ] && shift 1 && processor="$1" + ;; + *) + print_error "Unknown option: $1" + exit 1 + ;; + esac + [ "$#" -gt 0 ] && shift 1 + done + tree="${tree:-$basedir}" + stagedir="${stagedir:-$target}" + readonly tree="${tree:+$(realpath "$tree")}" + readonly stagedir="${stagedir:+$(realpath "$stagedir")}" + readonly basedir="${basedir:+$(realpath "$basedir")}" + readonly build_env="${build_env:-development}" + readonly command + if [ "$languages" ]; then + readonly languages="$(echo "$languages" | tr ',' ' ')" + else + readonly languages="$(ls -xw0 "${basedir}/global/languages")" + fi + if [ "$stagedir" != "$target" ] && printf %s "$target" | egrep -q '^.+@.+:(.+)?$'; then + readonly target + else + readonly target="${target:+$(realpath "$target")}" + fi - case "$command" in - build_into) [ -z "$target" ] && die "Missing destination directory" ;; - git_build_into) [ -z "$target" ] && die "Missing destination directory" ;; - build_run) [ -z "$target" ] && die "Missing destination directory" ;; - process_file) [ -z "$workfile" ] && die "Need at least input file" ;; - build_xmlstream) [ -z "$workfile" ] && die "Missing xhtml file name" ;; - tree_maker) [ -z "$target" ] && die "Missing target location" ;; - *help*) cat "$basedir/build/HELP"; exit 0 ;; - *) die "Urecognised command or no command given" ;; - esac + case "$command" in + build_into) [ -z "$target" ] && die "Missing destination directory" ;; + git_build_into) [ -z "$target" ] && die "Missing destination directory" ;; + build_run) [ -z "$target" ] && die "Missing destination directory" ;; + process_file) [ -z "$workfile" ] && die "Need at least input file" ;; + build_xmlstream) [ -z "$workfile" ] && die "Missing xhtml file name" ;; + tree_maker) [ -z "$target" ] && die "Missing target location" ;; + *help*) + cat "$basedir/build/README.md" + exit 0 + ;; + *) die "Urecognised command or no command given" ;; + esac fi diff --git a/build/build_main.sh b/build/build_main.sh index d3f8537314..d8b306e7b1 100755 --- a/build/build_main.sh +++ b/build/build_main.sh @@ -2,31 +2,31 @@ # Dependency check function check_dependencies() { - depends="$@" - deperrors='' - for depend in $depends; do - if ! which "$depend" >/dev/null 2>&1; then - deperrors="$depend $deperrors" - fi - done - if [ -n "$deperrors" ]; then - printf '\033[1;31m' - cat <<-EOF - The build script depends on some other programs to function. - Not all of those programs could be located on your system. - Please use your package manager to install the following programs: + depends="$@" + deperrors='' + for depend in $depends; do + if ! which "$depend" >/dev/null 2>&1; then + deperrors="$depend $deperrors" + fi + done + if [ -n "$deperrors" ]; then + printf '\033[1;31m' + cat <<-EOF + The build script depends on some other programs to function. + Not all of those programs could be located on your system. + Please use your package manager to install the following programs: EOF - printf '\n\033[0;31m%s\n' "$deperrors" - exit 1 - fi 1>&2 + printf '\n\033[0;31m%s\n' "$deperrors" + exit 1 + fi 1>&2 } # Check dependencies for all kinds of build envs (e.g. development, fsfe.org) check_dependencies realpath rsync xsltproc xmllint sed find egrep grep wc make tee date iconv wget shuf python3 if ! make --version | grep -q "GNU Make 4"; then - echo "The build script requires GNU Make 4.x" - exit 1 + echo "The build script requires GNU Make 4.x" + exit 1 fi basedir="${0%/*}/.." @@ -37,46 +37,46 @@ readonly start_time="$(date +%s)" # Check special dependencies for (test.)fsfe.org build server if [ "$build_env" == "fsfe.org" ] || [ "$build_env" == "test.fsfe.org" ]; then - check_dependencies lessc + check_dependencies lessc fi statusdir="${statusdir/#\~/$HOME}" if [ -n "$statusdir" ]; then - mkdir -p "$statusdir" - [ ! -w "$statusdir" -o ! -d "$statusdir" ] && \ - die "Unable to set up status directory in \"$statusdir\",\n" \ - "either select a status directory that exists and is writable,\n" \ - "or run the build script without output to a status directory" + mkdir -p "$statusdir" + [ ! -w "$statusdir" -o ! -d "$statusdir" ] && + die "Unable to set up status directory in \"$statusdir\",\n" \ + "either select a status directory that exists and is writable,\n" \ + "or run the build script without output to a status directory" fi -readonly statusdir="${statusdir:+$(realpath "$statusdir")}" +readonly statusdir="${statusdir:+$(realpath "$statusdir")}" buildpids=$( - ps -eo command \ - | egrep "[s]h ${0} .*" \ - | wc -l + ps -eo command | + egrep "[s]h ${0} .*" | + wc -l ) if [ $command = "build_into" -o $command = "git_build_into" ] && [ "$buildpids" -gt 2 ]; then - debug "build script is already running" - exit 1 + debug "build script is already running" + exit 1 fi [ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh" -[ -z "$inc_buildrun" ] && . "$basedir/build/buildrun.sh" +[ -z "$inc_buildrun" ] && . "$basedir/build/buildrun.sh" [ -z "$inc_makerules" ] && . "$basedir/build/makerules.sh" [ -z "$inc_processor" ] && . "$basedir/build/processor.sh" -[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh" +[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh" case "$command" in - git_build_into) if [ -f "${statusdir}/full_build" ]; then - debug "discovered flag file, performing full build" - rm "${statusdir}/full_build" - build_into - else - git_build_into - fi ;; - build_into) build_into ;; - build_run) buildrun ;; - process_file) process_file "$workfile" "$processor" ;; - build_xmlstream) build_xmlstream "$(get_shortname "$workfile")" "$(get_language "$workfile")" ;; - tree_maker) tree_maker "$tree" "$target" ;; +git_build_into) if [ -f "${statusdir}/full_build" ]; then + debug "discovered flag file, performing full build" + rm "${statusdir}/full_build" + build_into +else + git_build_into +fi ;; +build_into) build_into ;; +build_run) buildrun ;; +process_file) process_file "$workfile" "$processor" ;; +build_xmlstream) build_xmlstream "$(get_shortname "$workfile")" "$(get_language "$workfile")" ;; +tree_maker) tree_maker "$tree" "$target" ;; esac diff --git a/build/buildrun.sh b/build/buildrun.sh index b6a4578079..8411332265 100755 --- a/build/buildrun.sh +++ b/build/buildrun.sh @@ -5,131 +5,136 @@ inc_buildrun=true [ -z "$inc_logging" ] && . "$basedir/build/logging.sh" [ -z "$inc_misc" ] && . "$basedir/build/misc.sh" -match(){ - printf %s "$1" | egrep -q "$2" +match() { + printf %s "$1" | egrep -q "$2" } -dir_maker(){ - # set up directory tree for output - # optimise by only issuing mkdir commands - # for leaf directories - input="${1%/}" - output="${2%/}" +dir_maker() { + # set up directory tree for output + # optimise by only issuing mkdir commands + # for leaf directories + input="${1%/}" + output="${2%/}" - curpath="$output" - find "$input" -depth -type d \ - -regex "$input/[a-z\.]+\.[a-z]+\(/.*\)?" \ - -printf '%P\n' \ - | while read -r filepath; do - oldpath="$curpath" - curpath="$output/$filepath/" - match "$oldpath" "^$curpath" || mkdir -p "$curpath" - done + curpath="$output" + find "$input" -depth -type d \ + -regex "$input/[a-z\.]+\.[a-z]+\(/.*\)?" \ + -printf '%P\n' | + while read -r filepath; do + oldpath="$curpath" + curpath="$output/$filepath/" + match "$oldpath" "^$curpath" || mkdir -p "$curpath" + done } # The actual build -buildrun(){ - set -o pipefail +buildrun() { + set -o pipefail - printf %s "$start_time" > "$(logname start_time)" + printf %s "$start_time" >"$(logname start_time)" - ncpu="$(grep -c ^processor /proc/cpuinfo)" + ncpu="$(grep -c ^processor /proc/cpuinfo)" - [ -f "$(logname lasterror)" ] && rm "$(logname lasterror)" - [ -f "$(logname debug)" ] && rm "$(logname debug)" + [ -f "$(logname lasterror)" ] && rm "$(logname lasterror)" + [ -f "$(logname debug)" ] && rm "$(logname debug)" - { - echo "Starting phase 1" \ - && make --silent --directory="$basedir" build_env="${build_env:-development}" 2>&1 \ - && echo "Finishing phase 1" \ - || die "Error during phase 1" - } | t_logstatus phase_1 || exit 1 + { + echo "Starting phase 1" && + make --silent --directory="$basedir" build_env="${build_env}" languages="$languages" 2>&1 && + echo "Finishing phase 1" || + die "Error during phase 1" + } | t_logstatus phase_1 || exit 1 - dir_maker "$basedir" "$stagedir" || exit 1 + dir_maker "$basedir" "$stagedir" || exit 1 - forcelog Makefile + forcelog Makefile - { - tree_maker "$basedir" "$stagedir" 2>&1 \ - || die "Error during phase 2 Makefile generation" - } > "$(logname Makefile)" || exit 1 + { + tree_maker "$basedir" "$stagedir" 2>&1 || + die "Error during phase 2 Makefile generation" + } >"$(logname Makefile)" || exit 1 - { - echo "Starting phase 2" \ - && make --silent --jobs=$ncpu --file="$(logname Makefile)" 2>&1 \ - && echo "Finishing phase 2" \ - || die "Error during phase 2" - } | t_logstatus phase_2 || exit 1 + { + echo "Starting phase 2" && + make --silent --jobs=$ncpu --file="$(logname Makefile)" 2>&1 && + echo "Finishing phase 2" || + die "Error during phase 2" + } | t_logstatus phase_2 || exit 1 - if [ "$stagedir" != "$target" ]; then - # rsync issues a "copying unsafe symlink" message for each of the "unsafe" - # symlinks which we copy while rsyncing. These messages are issued even if - # the files have not changed and clutter up the output, so we filter them - # out. - { - for destination in ${target//,/ }; do - echo "Syncing files to $(echo "$destination" | grep -Po "(?<=@)[^:]+")" - rsync -av --copy-unsafe-links --del --exclude "status.fsfe.org/*fsfe.org/data" "$stagedir/" "$destination/" | grep -v "copying unsafe symlink" - done - } | t_logstatus stagesync - fi + if [ "$stagedir" != "$target" ]; then + # rsync issues a "copying unsafe symlink" message for each of the "unsafe" + # symlinks which we copy while rsyncing. These messages are issued even if + # the files have not changed and clutter up the output, so we filter them + # out. + { + for destination in ${target//,/ }; do + echo "Syncing files to $(echo "$destination" | grep -Po "(?<=@)[^:]+")" + rsync -av --copy-unsafe-links --del --exclude "status.fsfe.org/*fsfe.org/data" "$stagedir/" "$destination/" | grep -v "copying unsafe symlink" + done + } | t_logstatus stagesync + fi - date +%s > "$(logname end_time)" + date +%s >"$(logname end_time)" - if [ -n "$statusdir" ]; then - ( cd "$statusdir"/..; ./index.cgi | tail -n+3 > "$statusdir"/status_$(date +%s).html ) - fi + if [ -n "$statusdir" ]; then + ( + cd "$statusdir"/.. + ./index.cgi | tail -n+3 >"$statusdir"/status_$(date +%s).html + ) + fi } # Update git (try 3x) and then do an actual build -git_build_into(){ - forcelog GITchanges; GITchanges="$(logname GITchanges)" - forcelog GITerrors; GITerrors="$(logname GITerrors)" +git_build_into() { + forcelog GITchanges + GITchanges="$(logname GITchanges)" + forcelog GITerrors + GITerrors="$(logname GITerrors)" - gitterm=1 - i=0 - while [[ ( $gitterm -ne 0 ) && ( $i -lt 3) ]]; do - ((i++)) + gitterm=1 + i=0 + while [[ ($gitterm -ne 0) && ($i -lt 3) ]]; do + ((i++)) - git -C "$basedir" pull >"$GITchanges" 2>"$GITerrors" - gitterm="$?" + git -C "$basedir" pull >"$GITchanges" 2>"$GITerrors" + gitterm="$?" - if [ $gitterm -ne 0 ]; then - debug "Git pull unsuccessful. Trying again in a few seconds." - sleep $(shuf -i 10-30 -n1) - fi - done + if [ $gitterm -ne 0 ]; then + debug "Git pull unsuccessful. Trying again in a few seconds." + sleep $(shuf -i 10-30 -n1) + fi + done - if [ "$gitterm" -ne 0 ]; then - debug "Three git pulls failed, hard resetting and repulling" - git -C "$basedir" reset --hard HEAD~50 >"$GITchanges" 2>"$GITerrors" - git -C "$basedir" pull >>"$GITchanges" 2>>"$GITerrors" - gitterm="$?" - fi + if [ "$gitterm" -ne 0 ]; then + debug "Three git pulls failed, hard resetting and repulling" + git -C "$basedir" reset --hard HEAD~50 >"$GITchanges" 2>"$GITerrors" + git -C "$basedir" pull >>"$GITchanges" 2>>"$GITerrors" + gitterm="$?" + fi - if [ "$gitterm" -ne 0 ]; then - die "GIT reported the following problem:\n$(cat "$GITerrors")" - fi + if [ "$gitterm" -ne 0 ]; then + die "GIT reported the following problem:\n$(cat "$GITerrors")" + fi - if egrep '^Already up[ -]to[ -]date' "$GITchanges"; then - debug "No changes to GIT:\n$(cat "$GITchanges")" - # Exit status should only be 0 if there was a successful build. - # So set it to 1 here. - exit 1 - fi + if egrep '^Already up[ -]to[ -]date' "$GITchanges"; then + debug "No changes to GIT:\n$(cat "$GITchanges")" + # Exit status should only be 0 if there was a successful build. + # So set it to 1 here. + exit 1 + fi - logstatus GITlatest < "$GITchanges" - buildrun + logstatus GITlatest <"$GITchanges" + buildrun } # Clean up everything and then do an actual (full) build -build_into(){ +build_into() { - # Clean up source directory. - git -C "${basedir}" clean -dxf --exclude=status.fsfe.org/translations/data + # Clean up source directory. + git -C "${basedir}" clean -dxf --exclude=status.fsfe.org/translations/data - # Remove old stage directory. - rm -rf "${stagedir}" + # Remove old stage directory. + rm -rf "${stagedir}" - buildrun + buildrun } diff --git a/build/filenames.sh b/build/filenames.sh index 74eb53a177..4e281a1db2 100755 --- a/build/filenames.sh +++ b/build/filenames.sh @@ -2,15 +2,15 @@ inc_filenames=true -get_language(){ - # extract language indicator from a given file name - echo "$(echo "$1" |sed -r 's:^.*\.([a-z]{2})\.[^\.]+$:\1:')"; +get_language() { + # extract language indicator from a given file name + echo "$(echo "$1" | sed -r 's:^.*\.([a-z]{2})\.[^\.]+$:\1:')" } -get_shortname(){ - # get shortened version of a given file name - # required for internal processing +get_shortname() { + # get shortened version of a given file name + # required for internal processing - #echo "$(echo "$1" | sed -r 's:\.[a-z]{2}.xhtml$::')"; - echo "${1%.??.xhtml}" + #echo "$(echo "$1" | sed -r 's:\.[a-z]{2}.xhtml$::')"; + echo "${1%.??.xhtml}" } diff --git a/build/logging.sh b/build/logging.sh index e799bc47ca..479e8ea581 100755 --- a/build/logging.sh +++ b/build/logging.sh @@ -2,42 +2,42 @@ inc_logging=true -logname(){ - name="$1" +logname() { + name="$1" - if [ -w "$statusdir" ] && touch "$statusdir/$name"; then - echo "$statusdir/$name" - elif echo "$forcedlog" |egrep -q "^${name}=.+"; then - echo "$forcedlog" \ - | sed -rn "s;^${name}=;;p" - else - echo /dev/null - fi + if [ -w "$statusdir" ] && touch "$statusdir/$name"; then + echo "$statusdir/$name" + elif echo "$forcedlog" | egrep -q "^${name}=.+"; then + echo "$forcedlog" | + sed -rn "s;^${name}=;;p" + else + echo /dev/null + fi } -forcelog(){ - name="$1" +forcelog() { + name="$1" - [ "$(logname "$name")" = "/dev/null" ] \ - && forcedlog="$forcedlog\n${name}=$(mktemp -t w3bldXXXXXXXXX --suffix $$)" + [ "$(logname "$name")" = "/dev/null" ] && + forcedlog="$forcedlog\n${name}=$(mktemp -t w3bldXXXXXXXXX --suffix $$)" } [ -z "$USER" ] && USER="$(whoami)" trap "trap - 0 2 3 6 9 15; find \"${TMPDIR:-/tmp}/\" -maxdepth 1 -user \"$USER\" -name \"w3bld*$$\" -delete" 0 2 3 6 9 15 -logstatus(){ - # pipeline atom to write data streams into a log file - tee "$(logname "$1")" +logstatus() { + # pipeline atom to write data streams into a log file + tee "$(logname "$1")" } -t_logstatus(){ - # pipeline atom to write data streams into a log file - while read line; do - printf "[$(date +%T)] %s\n" "$line" - done |logstatus "$@" +t_logstatus() { + # pipeline atom to write data streams into a log file + while read line; do + printf "[$(date +%T)] %s\n" "$line" + done | logstatus "$@" } -logappend(){ - # pipeline atom to write data streams into a log file - tee -a "$(logname "$1")" +logappend() { + # pipeline atom to write data streams into a log file + tee -a "$(logname "$1")" } diff --git a/build/makerules.sh b/build/makerules.sh index d949153bbb..14257b5b57 100755 --- a/build/makerules.sh +++ b/build/makerules.sh @@ -2,13 +2,12 @@ inc_makerules=true -tree_maker(){ - # walk through file tree and issue Make rules according to file type - input="$(realpath "$1")" - output="$(realpath "$2")" - languages="$(ls -xw0 "${basedir}/global/languages")" +tree_maker() { + # walk through file tree and issue Make rules according to file type + input="$(realpath "$1")" + output="$(realpath "$2")" - cat < "\$@" EOF - done + done - cat <.html to file.html. @@ -110,15 +109,15 @@ HTML_DST_LINKS := \$(foreach base,\$(HTML_DST_BASES) \$(addsuffix index,\$(INDEX all: \$(HTML_DST_LINKS) EOF - for lang in ${languages}; do - cat< "\$@" EOF - done + done - cat < "\$@" EOF - done + done - cat <&2 - else - logappend debug >&2 - fi +debug() { + if [ "$#" -ge 1 ]; then + echo "$(date '+%F %T'): $@" | logappend debug >&2 + else + logappend debug >&2 + fi } -print_error(){ - echo "Error - $@" |logappend lasterror >&2 - echo "Run '$0 --help' to see usage instructions" >&2 +print_error() { + echo "Error - $@" | logappend lasterror >&2 + echo "Run '$0 --help' to see usage instructions" >&2 } -die(){ - echo "$(date '+%F %T'): Fatal - $@" |logappend lasterror >&2 - date +%s |logstatus end_time - exit 1 +die() { + echo "$(date '+%F %T'): Fatal - $@" | logappend lasterror >&2 + date +%s | logstatus end_time + exit 1 } diff --git a/build/process_file.sh b/build/process_file.sh index 3661d080a2..95935cbab6 100755 --- a/build/process_file.sh +++ b/build/process_file.sh @@ -6,6 +6,6 @@ basedir="${0%/*}/.." . "$basedir/build/arguments.sh" case "$command" in - process_file) process_file "$workfile" "$processor" ;; - *) die "Unrecognised command or no command given" ;; +process_file) process_file "$workfile" "$processor" ;; +*) die "Unrecognised command or no command given" ;; esac diff --git a/build/processor.sh b/build/processor.sh index 6f5a85449c..4d8c20a29c 100755 --- a/build/processor.sh +++ b/build/processor.sh @@ -2,48 +2,46 @@ inc_processor=true [ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh" -[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh" +[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh" -process_file(){ - infile="$1" - processor="$2" +process_file() { + infile="$1" + processor="$2" - shortname=$(get_shortname "$infile") - lang=$(get_language "$infile") + shortname=$(get_shortname "$infile") + lang=$(get_language "$infile") - if [ -z "${processor}" ]; then - if [ -f "${shortname}.xsl" ]; then - processor="${shortname}.xsl" - else - # Actually use the symlink target, so the relative includes are searched - # in the correct directory. - processor="$(realpath "${shortname%/*}/.default.xsl")" - fi - fi + if [ -z "${processor}" ]; then + if [ -f "${shortname}.xsl" ]; then + processor="${shortname}.xsl" + else + # Actually use the symlink target, so the relative includes are searched + # in the correct directory. + processor="$(realpath "${shortname%/*}/.default.xsl")" + fi + fi - # Make sure that the following pipe exits with a nonzero exit code if *any* - # of the commands fails. - set -o pipefail + # Make sure that the following pipe exits with a nonzero exit code if *any* + # of the commands fails. + set -o pipefail - # The sed command of death below does the following: - # 1. Remove https://fsfe.org (or https://test.fsfe.org) from the start of all - # links - # 2. Change links from /foo/bar.html into /foo/bar.xx.html - # 3. Change links from foo/bar.html into foo/bar.xx.html - # 4. Same for .rss and .ics links - # 5. Change links from /foo/bar/ into /foo/bar/index.xx.html - # 6. Change links from foo/bar/ into foo/bar/index.xx.html - # ... where xx is the language code. - # Everything is duplicated to allow for the href attribute to be enclosed in - # single or double quotes. - # I am strongly convinced that there must be a less obfuscated way of doing - # this. --Reinhard + # The sed command of death below does the following: + # 1. Remove https://fsfe.org (or https://test.fsfe.org) from the start of all + # links + # 2. Change links from /foo/bar.html into /foo/bar.xx.html + # 3. Change links from foo/bar.html into foo/bar.xx.html + # 4. Same for .rss and .ics links + # 5. Change links from /foo/bar/ into /foo/bar/index.xx.html + # 6. Change links from foo/bar/ into foo/bar/index.xx.html + # ... where xx is the language code. + # Everything is duplicated to allow for the href attribute to be enclosed in + # single or double quotes. + # I am strongly convinced that there must be a less obfuscated way of doing + # this. --Reinhard - build_xmlstream "$shortname" "$lang" \ - | xsltproc --stringparam "build-env" "${build_env:-development}" "$processor" - \ - | sed -r ':X; N; $!bX; - s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="(https?:)?//'"$domain"'/([^"]*)";<\1\2 href="/\4";gI - s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href='\''(https?:)?//'"$domain"'/([^'\'']*)'\'';<\1\2 href='\''/\4'\'';gI + build_xmlstream "$shortname" "$lang" | + xsltproc --stringparam "build-env" "${build_env}" "$processor" - | + sed -r ':X; N; $!bX; s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="((https?:)?//[^"]*)";<\1\2 href="#== norewrite ==\3";gI s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="([^#"])([^"]*/)?([^\./"]*\.)(html|rss|ics)(#[^"]*)?";<\1\2 href="\3\4\5'"$lang"'.\6\7";gI diff --git a/build/scaffold.sh b/build/scaffold.sh index e017d6c464..38bec31011 100755 --- a/build/scaffold.sh +++ b/build/scaffold.sh @@ -2,124 +2,124 @@ inc_scaffold=true -get_version(){ - version=$(xsltproc $basedir/build/xslt/get_version.xsl $1) - echo ${version:-0} +get_version() { + version=$(xsltproc $basedir/build/xslt/get_version.xsl $1) + echo ${version:-0} } -include_xml(){ - # include second level elements of a given XML file - # this emulates the behaviour of the original - # build script which wasn't able to load top - # level elements from any file - if [ -f "$1" ]; then - # Remove because the filename attribute would otherwise be added - # to this element instead of the actual content element. - sed 's;.*;;' "$1" | \ - sed -r ':X; $bY; N; bX; :Y; +include_xml() { + # include second level elements of a given XML file + # this emulates the behaviour of the original + # build script which wasn't able to load top + # level elements from any file + if [ -f "$1" ]; then + # Remove because the filename attribute would otherwise be added + # to this element instead of the actual content element. + sed 's;.*;;' "$1" | + sed -r ':X; $bY; N; bX; :Y; s:<(\?[xX][mM][lL]|!DOCTYPE)[[:space:]]+[^>]+>::g s:<[^!][^>]*>::; s:]*>([^<]*((<[^>]+/>|]|<[^>]*>)*>|<\?[^>]+>)[^<]*)*)?$:\1:;' - fi + fi } -get_attributes(){ - # get attributes of top level element in a given - # XHTML file - sed -rn ':X; N; $!bX; +get_attributes() { + # get attributes of top level element in a given + # XHTML file + sed -rn ':X; N; $!bX; s;^.*<[\n\t\r ]*([xX]|[xX]?[hH][tT])[mM][lL][\n\t\r ]+([^>]*)>.*$;\2;p' "$1" } -list_langs(){ - # list all languages a file exists in by globbing up - # the shortname (i.e. file path with file ending omitted) - # output is readily formatted for inclusion - # in xml stream - for file in "${1}".[a-z][a-z].xhtml; do - language="${file: -8:2}" - text="$(echo -n $(cat "${basedir}/global/languages/${language}"))" - echo "${text}" - done +list_langs() { + # list all languages a file exists in by globbing up + # the shortname (i.e. file path with file ending omitted) + # output is readily formatted for inclusion + # in xml stream + for file in "${1}".[a-z][a-z].xhtml; do + language="${file: -8:2}" + text="$(echo -n $(cat "${basedir}/global/languages/${language}"))" + echo "${text}" + done } -auto_sources(){ - # import elements from source files, add file name - # attribute to first element included from each file - shortname="$1" - lang="$2" +auto_sources() { + # import elements from source files, add file name + # attribute to first element included from each file + shortname="$1" + lang="$2" - list_file="$(dirname ${shortname})/.$(basename ${shortname}).xmllist" + list_file="$(dirname ${shortname})/.$(basename ${shortname}).xmllist" - if [ -f "${list_file}" ]; then - cat "${list_file}" | while read path; do - base="$(basename ${path})" - if [ -f "${basedir}/${path}.${lang}.xml" ]; then - printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.${lang}.xml")" - elif [ -f "${basedir}/${path}.en.xml" ]; then - printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.en.xml")" - fi - done \ - | sed -r ':X; N; $!bX; + if [ -f "${list_file}" ]; then + cat "${list_file}" | while read path; do + base="$(basename ${path})" + if [ -f "${basedir}/${path}.${lang}.xml" ]; then + printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.${lang}.xml")" + elif [ -f "${basedir}/${path}.en.xml" ]; then + printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.en.xml")" + fi + done | + sed -r ':X; N; $!bX; s;\n### (filename="[^\n"]+") ###\n[^<]*(]+>[^<]*)*(<([^/>]+/)*([^/>]+))(/?>);\2\3 \1\6;g;' - fi + fi } -build_xmlstream(){ - # assemble the xml stream for feeding into xsltproc - # the expected shortname and language flag indicate - # a single xhtml page to be built - shortname="$1" - lang="$2" +build_xmlstream() { + # assemble the xml stream for feeding into xsltproc + # the expected shortname and language flag indicate + # a single xhtml page to be built + shortname="$1" + lang="$2" - olang="$(echo "${shortname}".[a-z][a-z].xhtml "${shortname}".[e]n.xhtml |sed -rn 's;^.*\.([a-z]{2})\.xhtml.*$;\1;p')" - dirname="${shortname%/*}/" - topbanner_xml="$basedir/global/data/topbanner/.topbanner.${lang}.xml" - texts_xml="$basedir/global/data/texts/.texts.${lang}.xml" - date="$(date +%Y-%m-%d)" - time="$(date +%H:%M:%S)" - - if [ -f "${shortname}.${lang}.xhtml" ]; then - act_lang="$lang" - translation_state="up-to-date" - [ $(get_version "${shortname}.${olang}.xhtml") -gt $(get_version "${shortname}.${lang}.xhtml") ] && translation_state="outdated" - [ $(( $(get_version "${shortname}.${olang}.xhtml") - 3 )) -gt $(get_version "${shortname}.${lang}.xhtml") ] && act_lang="$olang" && translation_state="very-outdated" - else - act_lang="$olang" - translation_state="untranslated" - fi + olang="$(echo "${shortname}".[a-z][a-z].xhtml "${shortname}".[e]n.xhtml | sed -rn 's;^.*\.([a-z]{2})\.xhtml.*$;\1;p')" + dirname="${shortname%/*}/" + topbanner_xml="$basedir/global/data/topbanner/.topbanner.${lang}.xml" + texts_xml="$basedir/global/data/texts/.texts.${lang}.xml" + date="$(date +%Y-%m-%d)" + time="$(date +%H:%M:%S)" - infile="${shortname}.${act_lang}.xhtml" + if [ -f "${shortname}.${lang}.xhtml" ]; then + act_lang="$lang" + translation_state="up-to-date" + [ $(get_version "${shortname}.${olang}.xhtml") -gt $(get_version "${shortname}.${lang}.xhtml") ] && translation_state="outdated" + [ $(($(get_version "${shortname}.${olang}.xhtml") - 3)) -gt $(get_version "${shortname}.${lang}.xhtml") ] && act_lang="$olang" && translation_state="very-outdated" + else + act_lang="$olang" + translation_state="untranslated" + fi - cat <<-EOF - + infile="${shortname}.${act_lang}.xhtml" - - $(list_langs "$shortname") - + cat <<-EOF + - $(include_xml "$topbanner_xml") - $(include_xml "$basedir/global/data/texts/texts.en.xml") - $(include_xml "$texts_xml") - - - - $(auto_sources "${shortname}" "$lang") - + + $(list_langs "$shortname") + - $(include_xml "$infile") - + $(include_xml "$topbanner_xml") + $(include_xml "$basedir/global/data/texts/texts.en.xml") + $(include_xml "$texts_xml") - + + + $(auto_sources "${shortname}" "$lang") + + + $(include_xml "$infile") + + + EOF } diff --git a/fsfe.org/events/Makefile b/fsfe.org/events/Makefile index 8e388287e7..5caedd555e 100644 --- a/fsfe.org/events/Makefile +++ b/fsfe.org/events/Makefile @@ -19,7 +19,7 @@ ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS)) ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS)) # Languages in which the template exists -ARCH_LANGS := $(suffix $(basename $(wildcard archive-template.??.xhtml))) +ARCH_LANGS := $(filter $(foreach lang,$(languages),.$(lang)),$(suffix $(basename $(wildcard archive-template.??.xhtml)))) # .xhtml files to generate ARCH_XHTML := $(foreach year,$(ARCH_YEARS),$(foreach lang,$(ARCH_LANGS),$(year)/index$(lang).xhtml)) diff --git a/fsfe.org/news/Makefile b/fsfe.org/news/Makefile index 3455cb87be..2f03621a51 100644 --- a/fsfe.org/news/Makefile +++ b/fsfe.org/news/Makefile @@ -19,7 +19,7 @@ ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS)) ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS)) # Languages in which the template exists -ARCH_LANGS := $(suffix $(basename $(wildcard archive-template.??.xhtml))) +ARCH_LANGS := $(filter $(foreach lang,$(languages),.$(lang)),$(suffix $(basename $(wildcard archive-template.??.xhtml)))) # .xhtml files to generate ARCH_XHTML := $(foreach year,$(ARCH_YEARS),$(foreach lang,$(ARCH_LANGS),$(year)/index$(lang).xhtml)) @@ -62,7 +62,7 @@ all: $(GENERATED_XML_SOURCES) # ----------------------------------------------------------------------------- # All existing .xhtml files -XHTML := $(shell ls */*.??.xhtml | xargs grep -l '> "${tagmaps}/${tag}"