feat/build-improvements-2 (#4553)
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
Another attempt at #4516 Adds a small fix for tag map generation #4516 introduces build failures, but testing indicates that those failures would have been resolved by a full rebuild. Hence this pr. Co-authored-by: Darragh Elliott <me@delliott.xyz> Reviewed-on: #4553 Co-authored-by: delliott <delliott@fsfe.org> Co-committed-by: delliott <delliott@fsfe.org>
This commit is contained in:
parent
fa6c9a7d7c
commit
bb45fc85ba
13
Makefile
13
Makefile
@ -12,6 +12,7 @@
|
||||
|
||||
# This will be overwritten in the command line running this Makefile.
|
||||
build_env = development
|
||||
languages = none
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Build search index
|
||||
@ -68,7 +69,7 @@ SUBDIRS := $(shell find . -regex "./[a-z\.]+\.[a-z]+/.*/Makefile" | xargs dirnam
|
||||
all: $(SUBDIRS)
|
||||
$(SUBDIRS): .FORCE
|
||||
echo "* Preparing subdirectory $@"
|
||||
$(MAKE) --silent --directory=$@
|
||||
$(MAKE) --silent --directory=$@ languages="$(languages)"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create XML symlinks
|
||||
@ -82,9 +83,7 @@ $(SUBDIRS): .FORCE
|
||||
# otherwise. This symlinks make sure that phase 2 can easily use the right file
|
||||
# for each language, also as a prerequisite in the Makefile.
|
||||
|
||||
LANGUAGES := $(shell ls -xw0 global/languages)
|
||||
|
||||
TEXTS_LINKS := $(foreach lang,$(LANGUAGES),global/data/texts/.texts.$(lang).xml)
|
||||
TEXTS_LINKS := $(foreach lang,$(languages),global/data/texts/.texts.$(lang).xml)
|
||||
|
||||
all: $(TEXTS_LINKS)
|
||||
global/data/texts/.texts.%.xml: .FORCE
|
||||
@ -94,7 +93,7 @@ global/data/texts/.texts.%.xml: .FORCE
|
||||
ln -sf texts.en.xml $@; \
|
||||
fi
|
||||
|
||||
TOPBANNER_LINKS := $(foreach lang,$(LANGUAGES),global/data/topbanner/.topbanner.$(lang).xml)
|
||||
TOPBANNER_LINKS := $(foreach lang,$(languages),global/data/topbanner/.topbanner.$(lang).xml)
|
||||
|
||||
all: $(TOPBANNER_LINKS)
|
||||
global/data/topbanner/.topbanner.%.xml: .FORCE
|
||||
@ -134,7 +133,7 @@ default_xsl:
|
||||
.PHONY: localmenus
|
||||
all: localmenus
|
||||
localmenus: $(SUBDIRS)
|
||||
tools/update_localmenus.sh
|
||||
tools/update_localmenus.sh "$(languages)"
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Update XML filelists
|
||||
@ -155,4 +154,4 @@ localmenus: $(SUBDIRS)
|
||||
.PHONY: xmllists
|
||||
all: xmllists
|
||||
xmllists: $(SUBDIRS)
|
||||
tools/update_xmllists.sh
|
||||
tools/update_xmllists.sh "$(languages)"
|
||||
|
||||
@ -72,7 +72,7 @@ You can see the current status of translation progress of fsfe.org at [status.fs
|
||||
|
||||
## Build
|
||||
|
||||
There are two ways to build and develop the directory locally. Initial builds of the webpages may take ~40 minutes, but subsequent builds should be much faster.
|
||||
There are two ways to build and develop the directory locally. Initial builds of the webpages may take ~40 minutes, but subsequent builds should be much faster. Using the `--languages` flag to avoid building all supported languages can make this much faster. See ./build/README.md for more information.
|
||||
|
||||
Alterations to build scripts or the files used site-wide will result in near full rebuilds.
|
||||
|
||||
|
||||
51
build.sh
51
build.sh
@ -2,26 +2,47 @@
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 -f|--full -s|--serve" 1>&2
|
||||
exit 1
|
||||
cat <<-EOF
|
||||
# build.sh Usage
|
||||
## General
|
||||
This script is a wrapper script over ./build/build_main.sh that provides nicer option names, and the options to serve the files.
|
||||
For documentation on the build script itself see ./build/README.md
|
||||
## Flags
|
||||
### -f | --full
|
||||
Perform a full rebuild of the webpages.
|
||||
### -s | --serve
|
||||
Serve the build webpages over localhost.
|
||||
### --
|
||||
Everything after this is passed directly to build_main.
|
||||
See ./build/README.md for valid options.
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
command="build_run"
|
||||
serve=""
|
||||
extra_args=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--full | -f)
|
||||
command="build_into" && shift 1
|
||||
;;
|
||||
--serve | -s)
|
||||
serve="true" && shift 1
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
case "$1" in
|
||||
--full | -f)
|
||||
command="build_into" && shift 1
|
||||
;;
|
||||
--serve | -s)
|
||||
serve="true" && shift 1
|
||||
;;
|
||||
--)
|
||||
shift 1
|
||||
while [ "$#" -gt 0 ]; do
|
||||
extra_args+="$1 "
|
||||
shift 1
|
||||
done
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
mkdir -p ./output
|
||||
./build/build_main.sh "$command" ./output/final --statusdir ./output/final/status.fsfe.org/fsfe.org/data
|
||||
./build/build_main.sh "$command" ./output/final --statusdir ./output/final/status.fsfe.org/fsfe.org/data $extra_args
|
||||
if [[ "$serve" ]]; then
|
||||
python3 ./serve-websites.py
|
||||
python3 ./serve-websites.py
|
||||
fi
|
||||
|
||||
58
build/HELP
58
build/HELP
@ -1,58 +0,0 @@
|
||||
|
||||
Usage:
|
||||
------------------------------------------------------------------------------
|
||||
|
||||
build_main.sh [options] build_into "destination_dir"
|
||||
Perform the page build. Write output to destination_dir. The source
|
||||
directory is determined from the build scripts own location.
|
||||
|
||||
build_main.sh [options] build_xmlstream "file.xhtml"
|
||||
Compile an xml stream from the specified file, additional sources will be
|
||||
determined and included automatically. The stream is suitable for being
|
||||
passed into xsltproc.
|
||||
|
||||
build_main.sh [options] process_file "file.xhtml" [processor.xsl]
|
||||
Generate output from an xhtml file as if it would be processed during the
|
||||
build. Output is written to STDOUT and can be redirected as desired.
|
||||
If a xslt file is not given, it will be chosen automatically.
|
||||
|
||||
build_main.sh [options] tree_maker [input_dir] "destination_dir"
|
||||
Generate a set of make rules to build the website contained in input_dir.
|
||||
destination_dir should be the www root of a web server.
|
||||
If input_dir is omitted, it will be the source directory determined from
|
||||
the build scripts location.
|
||||
Note: if destination_dir is set via previous options, and only one parameter
|
||||
is given, then this parameter will be interpreted as input_dir
|
||||
|
||||
OPTIONS
|
||||
-------
|
||||
|
||||
--source "source_dir"
|
||||
Force a specific source directory. If not explicitly given source_dir is
|
||||
determined from the build scripts own location.
|
||||
Paths given in .sources files are interpreted as relative to source_dir
|
||||
making this option useful when building a webpage outside of the build
|
||||
scripts "regular" tree.
|
||||
|
||||
--destination "destination_dir"
|
||||
The directory into which the website will be built. This option can be used
|
||||
in conjunction with the tree_maker and build_into commands. It will override
|
||||
the destination_dir option given after those commands and is therefore
|
||||
redundant. The option exists to provide backward compatibility to the 2002
|
||||
build script.
|
||||
|
||||
--statusdir "status_dir"
|
||||
A directory to which messages are written. If no status_dir is provided
|
||||
information will be written to stdout. The directory will also be used
|
||||
to store some temporary files, which would otherwise be set up in the
|
||||
system wide temp directory.
|
||||
|
||||
--build-env "selection"
|
||||
Indicate the current build environment. "selection" can be one of:
|
||||
* "fsfe.org": building https://fsfe.org on the production server
|
||||
* "test.fsfe.org": building https://test.fsfe.org on the production server
|
||||
* "development" (default): local development build
|
||||
In a local development build, code to dynamically compile the less files into
|
||||
CSS will be included in the HTML output, while in the other environments, the
|
||||
precompiles fsfe.min.css (or valentine.min.css) will be referenced from the
|
||||
generated web pages.
|
||||
42
build/README.md
Normal file
42
build/README.md
Normal file
@ -0,0 +1,42 @@
|
||||
# Main Commands
|
||||
Note that targets takes a comma separated list of valid rsync targets, and hence supports ssh targets. If targeting more than one directory one must use the --stage-dir flag documented below.
|
||||
## build_main.sh [options] build_run "targets"
|
||||
Perform the page build. Write output to targets. The source directory is determined from the build scripts own location.
|
||||
|
||||
## build_main.sh [options] git_build_into "targets"
|
||||
Update repo to latest version of upstream branch and then perform a standard build. Write output to targets. The source directory is determined from the build scripts own location.
|
||||
|
||||
## build_main.sh [options] build_into "targets"
|
||||
Perform a full rebuild of the webpages, removing all cached files. Write output to targets. The source directory is determined from the build scripts own location.
|
||||
|
||||
# Internal Commands
|
||||
It is unlikely that you will need to directly call these commands, but they are documented here never the less.
|
||||
## build_main.sh [options] build_xmlstream "file.xhtml"
|
||||
Compile an xml stream from the specified file, additional sources will be determined and included automatically. The stream is suitable for being passed into xsltproc.
|
||||
|
||||
## build_main.sh [options] process_file "file.xhtml" [processor.xsl]
|
||||
Generate output from an xhtml file as if it would be processed during the
|
||||
build. Output is written to STDOUT and can be redirected as desired.
|
||||
If a xslt file is not given, it will be chosen automatically.
|
||||
|
||||
## build_main.sh [options] tree_maker [input_dir] "targets"
|
||||
Generate a set of make rules to build the website contained in input_dir. targets should be the www root of a web server. If input_dir is omitted, it will be the source directory determined from the build scripts location. Note: if targets is set via previous options, and only one parameter is given, then this parameter will be interpreted as input_dir
|
||||
|
||||
# OPTIONS
|
||||
## --source "source_dir"
|
||||
Force a specific source directory. If not explicitly given source_dir is determined from the build scripts own location. Paths given in .sources files are interpreted as relative to source_dir making this option useful when building a webpage outside of the build scripts "regular" tree.
|
||||
|
||||
## --status-dir "status_dir"
|
||||
A directory to which messages are written. If no status_dir is provided information will be written to stdout. The directory will also be used to store some temporary files, which would otherwise be set up in the system wide temp directory.
|
||||
|
||||
## --stage-dir "stage_dir"
|
||||
Directory used for staging the website builds. The websites are first build into this directory, then copied to each targets.
|
||||
|
||||
## --build-env "selection"
|
||||
Indicate the current build environment. "selection" can be one of: * "fsfe.org": building https://fsfe.org on the production server * "test.fsfe.org": building https://test.fsfe.org on the production server * "development" (default): local development build In a local development build, code to dynamically compile the less files into CSS will be included in the HTML output, while in the other environments, the precompiles fsfe.min.css (or valentine.min.css) will be referenced from the generated web pages.
|
||||
|
||||
## --languages "languages"
|
||||
Takes a comma separated list of language shot codes to build the website in. For example, to build the site in English and French only one would use `--languages en,fr`. One of the built languages must be English.
|
||||
|
||||
## --help
|
||||
Show this README.
|
||||
@ -3,87 +3,92 @@
|
||||
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
|
||||
|
||||
if [ -z "$inc_arguments" ]; then
|
||||
inc_arguments=true
|
||||
basedir="$(realpath "${0%/*}/..")"
|
||||
inc_arguments=true
|
||||
basedir="$(realpath "${0%/*}/..")"
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
-s|--statusdir|--status-dir)
|
||||
[ "$#" -gt 0 ] && shift 1 && statusdir="$1"
|
||||
;;
|
||||
--domain)
|
||||
[ "$#" -gt 0 ] && shift 1 && domain="$1"
|
||||
;;
|
||||
--source)
|
||||
[ "$#" -gt 0 ] && shift 1 && basedir="$1"
|
||||
;;
|
||||
--stage|--stagedir)
|
||||
[ "$#" -gt 0 ] && shift 1 && stagedir="$1"
|
||||
;;
|
||||
-d|--dest|--destination)
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
--build-env)
|
||||
[ "$#" -gt 0 ] && shift 1 && build_env="$1"
|
||||
;;
|
||||
-h|--help)
|
||||
command="help"
|
||||
;;
|
||||
build_into)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
git_build_into)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
build_run)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
build_xmlstream)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && workfile="$1"
|
||||
;;
|
||||
tree_maker)
|
||||
command="$1$command"
|
||||
[ -n "$target" -o -n "$3" ] && shift 1 && tree="$1"
|
||||
shift 1; [ -n "$1" ] && target="$1"
|
||||
;;
|
||||
process_file)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && workfile="$1"
|
||||
[ "$#" -gt 0 ] && shift 1 && processor="$1"
|
||||
;;
|
||||
*)
|
||||
print_error "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
[ "$#" -gt 0 ] && shift 1
|
||||
done
|
||||
|
||||
tree="${tree:-$basedir}"
|
||||
stagedir="${stagedir:-$target}"
|
||||
readonly tree="${tree:+$(realpath "$tree")}"
|
||||
readonly stagedir="${stagedir:+$(realpath "$stagedir")}"
|
||||
readonly basedir="${basedir:+$(realpath "$basedir")}"
|
||||
readonly domain="${domain:-www.fsfe.org}"
|
||||
readonly command
|
||||
if [ "$stagedir" != "$target" ] && printf %s "$target" | egrep -q '^.+@.+:(.+)?$'; then
|
||||
readonly target
|
||||
else
|
||||
readonly target="${target:+$(realpath "$target")}"
|
||||
fi
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
-s | --statusdir | --status-dir)
|
||||
[ "$#" -gt 0 ] && shift 1 && statusdir="$1"
|
||||
;;
|
||||
--source)
|
||||
[ "$#" -gt 0 ] && shift 1 && basedir="$1"
|
||||
;;
|
||||
--stage | --stagedir | --stage-dir)
|
||||
[ "$#" -gt 0 ] && shift 1 && stagedir="$1"
|
||||
;;
|
||||
--build-env)
|
||||
[ "$#" -gt 0 ] && shift 1 && build_env="$1"
|
||||
;;
|
||||
--languages)
|
||||
[ "$#" -gt 0 ] && shift 1 && languages="$1"
|
||||
;;
|
||||
-h | --help)
|
||||
command="help"
|
||||
;;
|
||||
build_into)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
git_build_into)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
build_run)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && target="$1"
|
||||
;;
|
||||
build_xmlstream)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && workfile="$1"
|
||||
;;
|
||||
tree_maker)
|
||||
command="$1$command"
|
||||
[ -n "$target" -o -n "$3" ] && shift 1 && tree="$1"
|
||||
shift 1
|
||||
[ -n "$1" ] && target="$1"
|
||||
;;
|
||||
process_file)
|
||||
command="$1$command"
|
||||
[ "$#" -gt 0 ] && shift 1 && workfile="$1"
|
||||
[ "$#" -gt 0 ] && shift 1 && processor="$1"
|
||||
;;
|
||||
*)
|
||||
print_error "Unknown option: $1"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
[ "$#" -gt 0 ] && shift 1
|
||||
done
|
||||
tree="${tree:-$basedir}"
|
||||
stagedir="${stagedir:-$target}"
|
||||
readonly tree="${tree:+$(realpath "$tree")}"
|
||||
readonly stagedir="${stagedir:+$(realpath "$stagedir")}"
|
||||
readonly basedir="${basedir:+$(realpath "$basedir")}"
|
||||
readonly build_env="${build_env:-development}"
|
||||
readonly command
|
||||
if [ "$languages" ]; then
|
||||
readonly languages="$(echo "$languages" | tr ',' ' ')"
|
||||
else
|
||||
readonly languages="$(ls -xw0 "${basedir}/global/languages")"
|
||||
fi
|
||||
if [ "$stagedir" != "$target" ] && printf %s "$target" | egrep -q '^.+@.+:(.+)?$'; then
|
||||
readonly target
|
||||
else
|
||||
readonly target="${target:+$(realpath "$target")}"
|
||||
fi
|
||||
|
||||
case "$command" in
|
||||
build_into) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
git_build_into) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
build_run) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
process_file) [ -z "$workfile" ] && die "Need at least input file" ;;
|
||||
build_xmlstream) [ -z "$workfile" ] && die "Missing xhtml file name" ;;
|
||||
tree_maker) [ -z "$target" ] && die "Missing target location" ;;
|
||||
*help*) cat "$basedir/build/HELP"; exit 0 ;;
|
||||
*) die "Urecognised command or no command given" ;;
|
||||
esac
|
||||
case "$command" in
|
||||
build_into) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
git_build_into) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
build_run) [ -z "$target" ] && die "Missing destination directory" ;;
|
||||
process_file) [ -z "$workfile" ] && die "Need at least input file" ;;
|
||||
build_xmlstream) [ -z "$workfile" ] && die "Missing xhtml file name" ;;
|
||||
tree_maker) [ -z "$target" ] && die "Missing target location" ;;
|
||||
*help*)
|
||||
cat "$basedir/build/README.md"
|
||||
exit 0
|
||||
;;
|
||||
*) die "Urecognised command or no command given" ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
@ -2,31 +2,31 @@
|
||||
|
||||
# Dependency check function
|
||||
check_dependencies() {
|
||||
depends="$@"
|
||||
deperrors=''
|
||||
for depend in $depends; do
|
||||
if ! which "$depend" >/dev/null 2>&1; then
|
||||
deperrors="$depend $deperrors"
|
||||
fi
|
||||
done
|
||||
if [ -n "$deperrors" ]; then
|
||||
printf '\033[1;31m'
|
||||
cat <<-EOF
|
||||
The build script depends on some other programs to function.
|
||||
Not all of those programs could be located on your system.
|
||||
Please use your package manager to install the following programs:
|
||||
depends="$@"
|
||||
deperrors=''
|
||||
for depend in $depends; do
|
||||
if ! which "$depend" >/dev/null 2>&1; then
|
||||
deperrors="$depend $deperrors"
|
||||
fi
|
||||
done
|
||||
if [ -n "$deperrors" ]; then
|
||||
printf '\033[1;31m'
|
||||
cat <<-EOF
|
||||
The build script depends on some other programs to function.
|
||||
Not all of those programs could be located on your system.
|
||||
Please use your package manager to install the following programs:
|
||||
EOF
|
||||
printf '\n\033[0;31m%s\n' "$deperrors"
|
||||
exit 1
|
||||
fi 1>&2
|
||||
printf '\n\033[0;31m%s\n' "$deperrors"
|
||||
exit 1
|
||||
fi 1>&2
|
||||
}
|
||||
|
||||
# Check dependencies for all kinds of build envs (e.g. development, fsfe.org)
|
||||
check_dependencies realpath rsync xsltproc xmllint sed find egrep grep wc make tee date iconv wget shuf python3
|
||||
|
||||
if ! make --version | grep -q "GNU Make 4"; then
|
||||
echo "The build script requires GNU Make 4.x"
|
||||
exit 1
|
||||
echo "The build script requires GNU Make 4.x"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
basedir="${0%/*}/.."
|
||||
@ -37,46 +37,46 @@ readonly start_time="$(date +%s)"
|
||||
|
||||
# Check special dependencies for (test.)fsfe.org build server
|
||||
if [ "$build_env" == "fsfe.org" ] || [ "$build_env" == "test.fsfe.org" ]; then
|
||||
check_dependencies lessc
|
||||
check_dependencies lessc
|
||||
fi
|
||||
|
||||
statusdir="${statusdir/#\~/$HOME}"
|
||||
if [ -n "$statusdir" ]; then
|
||||
mkdir -p "$statusdir"
|
||||
[ ! -w "$statusdir" -o ! -d "$statusdir" ] && \
|
||||
die "Unable to set up status directory in \"$statusdir\",\n" \
|
||||
"either select a status directory that exists and is writable,\n" \
|
||||
"or run the build script without output to a status directory"
|
||||
mkdir -p "$statusdir"
|
||||
[ ! -w "$statusdir" -o ! -d "$statusdir" ] &&
|
||||
die "Unable to set up status directory in \"$statusdir\",\n" \
|
||||
"either select a status directory that exists and is writable,\n" \
|
||||
"or run the build script without output to a status directory"
|
||||
fi
|
||||
readonly statusdir="${statusdir:+$(realpath "$statusdir")}"
|
||||
readonly statusdir="${statusdir:+$(realpath "$statusdir")}"
|
||||
|
||||
buildpids=$(
|
||||
ps -eo command \
|
||||
| egrep "[s]h ${0} .*" \
|
||||
| wc -l
|
||||
ps -eo command |
|
||||
egrep "[s]h ${0} .*" |
|
||||
wc -l
|
||||
)
|
||||
if [ $command = "build_into" -o $command = "git_build_into" ] && [ "$buildpids" -gt 2 ]; then
|
||||
debug "build script is already running"
|
||||
exit 1
|
||||
debug "build script is already running"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh"
|
||||
[ -z "$inc_buildrun" ] && . "$basedir/build/buildrun.sh"
|
||||
[ -z "$inc_buildrun" ] && . "$basedir/build/buildrun.sh"
|
||||
[ -z "$inc_makerules" ] && . "$basedir/build/makerules.sh"
|
||||
[ -z "$inc_processor" ] && . "$basedir/build/processor.sh"
|
||||
[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh"
|
||||
[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh"
|
||||
|
||||
case "$command" in
|
||||
git_build_into) if [ -f "${statusdir}/full_build" ]; then
|
||||
debug "discovered flag file, performing full build"
|
||||
rm "${statusdir}/full_build"
|
||||
build_into
|
||||
else
|
||||
git_build_into
|
||||
fi ;;
|
||||
build_into) build_into ;;
|
||||
build_run) buildrun ;;
|
||||
process_file) process_file "$workfile" "$processor" ;;
|
||||
build_xmlstream) build_xmlstream "$(get_shortname "$workfile")" "$(get_language "$workfile")" ;;
|
||||
tree_maker) tree_maker "$tree" "$target" ;;
|
||||
git_build_into) if [ -f "${statusdir}/full_build" ]; then
|
||||
debug "discovered flag file, performing full build"
|
||||
rm "${statusdir}/full_build"
|
||||
build_into
|
||||
else
|
||||
git_build_into
|
||||
fi ;;
|
||||
build_into) build_into ;;
|
||||
build_run) buildrun ;;
|
||||
process_file) process_file "$workfile" "$processor" ;;
|
||||
build_xmlstream) build_xmlstream "$(get_shortname "$workfile")" "$(get_language "$workfile")" ;;
|
||||
tree_maker) tree_maker "$tree" "$target" ;;
|
||||
esac
|
||||
|
||||
@ -5,131 +5,136 @@ inc_buildrun=true
|
||||
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
|
||||
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
|
||||
|
||||
match(){
|
||||
printf %s "$1" | egrep -q "$2"
|
||||
match() {
|
||||
printf %s "$1" | egrep -q "$2"
|
||||
}
|
||||
|
||||
dir_maker(){
|
||||
# set up directory tree for output
|
||||
# optimise by only issuing mkdir commands
|
||||
# for leaf directories
|
||||
input="${1%/}"
|
||||
output="${2%/}"
|
||||
dir_maker() {
|
||||
# set up directory tree for output
|
||||
# optimise by only issuing mkdir commands
|
||||
# for leaf directories
|
||||
input="${1%/}"
|
||||
output="${2%/}"
|
||||
|
||||
curpath="$output"
|
||||
find "$input" -depth -type d \
|
||||
-regex "$input/[a-z\.]+\.[a-z]+\(/.*\)?" \
|
||||
-printf '%P\n' \
|
||||
| while read -r filepath; do
|
||||
oldpath="$curpath"
|
||||
curpath="$output/$filepath/"
|
||||
match "$oldpath" "^$curpath" || mkdir -p "$curpath"
|
||||
done
|
||||
curpath="$output"
|
||||
find "$input" -depth -type d \
|
||||
-regex "$input/[a-z\.]+\.[a-z]+\(/.*\)?" \
|
||||
-printf '%P\n' |
|
||||
while read -r filepath; do
|
||||
oldpath="$curpath"
|
||||
curpath="$output/$filepath/"
|
||||
match "$oldpath" "^$curpath" || mkdir -p "$curpath"
|
||||
done
|
||||
}
|
||||
|
||||
# The actual build
|
||||
buildrun(){
|
||||
set -o pipefail
|
||||
buildrun() {
|
||||
set -o pipefail
|
||||
|
||||
printf %s "$start_time" > "$(logname start_time)"
|
||||
printf %s "$start_time" >"$(logname start_time)"
|
||||
|
||||
ncpu="$(grep -c ^processor /proc/cpuinfo)"
|
||||
ncpu="$(grep -c ^processor /proc/cpuinfo)"
|
||||
|
||||
[ -f "$(logname lasterror)" ] && rm "$(logname lasterror)"
|
||||
[ -f "$(logname debug)" ] && rm "$(logname debug)"
|
||||
[ -f "$(logname lasterror)" ] && rm "$(logname lasterror)"
|
||||
[ -f "$(logname debug)" ] && rm "$(logname debug)"
|
||||
|
||||
{
|
||||
echo "Starting phase 1" \
|
||||
&& make --silent --directory="$basedir" build_env="${build_env:-development}" 2>&1 \
|
||||
&& echo "Finishing phase 1" \
|
||||
|| die "Error during phase 1"
|
||||
} | t_logstatus phase_1 || exit 1
|
||||
{
|
||||
echo "Starting phase 1" &&
|
||||
make --silent --directory="$basedir" build_env="${build_env}" languages="$languages" 2>&1 &&
|
||||
echo "Finishing phase 1" ||
|
||||
die "Error during phase 1"
|
||||
} | t_logstatus phase_1 || exit 1
|
||||
|
||||
dir_maker "$basedir" "$stagedir" || exit 1
|
||||
dir_maker "$basedir" "$stagedir" || exit 1
|
||||
|
||||
forcelog Makefile
|
||||
forcelog Makefile
|
||||
|
||||
{
|
||||
tree_maker "$basedir" "$stagedir" 2>&1 \
|
||||
|| die "Error during phase 2 Makefile generation"
|
||||
} > "$(logname Makefile)" || exit 1
|
||||
{
|
||||
tree_maker "$basedir" "$stagedir" 2>&1 ||
|
||||
die "Error during phase 2 Makefile generation"
|
||||
} >"$(logname Makefile)" || exit 1
|
||||
|
||||
{
|
||||
echo "Starting phase 2" \
|
||||
&& make --silent --jobs=$ncpu --file="$(logname Makefile)" 2>&1 \
|
||||
&& echo "Finishing phase 2" \
|
||||
|| die "Error during phase 2"
|
||||
} | t_logstatus phase_2 || exit 1
|
||||
{
|
||||
echo "Starting phase 2" &&
|
||||
make --silent --jobs=$ncpu --file="$(logname Makefile)" 2>&1 &&
|
||||
echo "Finishing phase 2" ||
|
||||
die "Error during phase 2"
|
||||
} | t_logstatus phase_2 || exit 1
|
||||
|
||||
if [ "$stagedir" != "$target" ]; then
|
||||
# rsync issues a "copying unsafe symlink" message for each of the "unsafe"
|
||||
# symlinks which we copy while rsyncing. These messages are issued even if
|
||||
# the files have not changed and clutter up the output, so we filter them
|
||||
# out.
|
||||
{
|
||||
for destination in ${target//,/ }; do
|
||||
echo "Syncing files to $(echo "$destination" | grep -Po "(?<=@)[^:]+")"
|
||||
rsync -av --copy-unsafe-links --del --exclude "status.fsfe.org/*fsfe.org/data" "$stagedir/" "$destination/" | grep -v "copying unsafe symlink"
|
||||
done
|
||||
} | t_logstatus stagesync
|
||||
fi
|
||||
if [ "$stagedir" != "$target" ]; then
|
||||
# rsync issues a "copying unsafe symlink" message for each of the "unsafe"
|
||||
# symlinks which we copy while rsyncing. These messages are issued even if
|
||||
# the files have not changed and clutter up the output, so we filter them
|
||||
# out.
|
||||
{
|
||||
for destination in ${target//,/ }; do
|
||||
echo "Syncing files to $(echo "$destination" | grep -Po "(?<=@)[^:]+")"
|
||||
rsync -av --copy-unsafe-links --del --exclude "status.fsfe.org/*fsfe.org/data" "$stagedir/" "$destination/" | grep -v "copying unsafe symlink"
|
||||
done
|
||||
} | t_logstatus stagesync
|
||||
fi
|
||||
|
||||
date +%s > "$(logname end_time)"
|
||||
date +%s >"$(logname end_time)"
|
||||
|
||||
if [ -n "$statusdir" ]; then
|
||||
( cd "$statusdir"/..; ./index.cgi | tail -n+3 > "$statusdir"/status_$(date +%s).html )
|
||||
fi
|
||||
if [ -n "$statusdir" ]; then
|
||||
(
|
||||
cd "$statusdir"/..
|
||||
./index.cgi | tail -n+3 >"$statusdir"/status_$(date +%s).html
|
||||
)
|
||||
fi
|
||||
}
|
||||
|
||||
# Update git (try 3x) and then do an actual build
|
||||
git_build_into(){
|
||||
forcelog GITchanges; GITchanges="$(logname GITchanges)"
|
||||
forcelog GITerrors; GITerrors="$(logname GITerrors)"
|
||||
git_build_into() {
|
||||
forcelog GITchanges
|
||||
GITchanges="$(logname GITchanges)"
|
||||
forcelog GITerrors
|
||||
GITerrors="$(logname GITerrors)"
|
||||
|
||||
gitterm=1
|
||||
i=0
|
||||
while [[ ( $gitterm -ne 0 ) && ( $i -lt 3) ]]; do
|
||||
((i++))
|
||||
gitterm=1
|
||||
i=0
|
||||
while [[ ($gitterm -ne 0) && ($i -lt 3) ]]; do
|
||||
((i++))
|
||||
|
||||
git -C "$basedir" pull >"$GITchanges" 2>"$GITerrors"
|
||||
gitterm="$?"
|
||||
git -C "$basedir" pull >"$GITchanges" 2>"$GITerrors"
|
||||
gitterm="$?"
|
||||
|
||||
if [ $gitterm -ne 0 ]; then
|
||||
debug "Git pull unsuccessful. Trying again in a few seconds."
|
||||
sleep $(shuf -i 10-30 -n1)
|
||||
fi
|
||||
done
|
||||
if [ $gitterm -ne 0 ]; then
|
||||
debug "Git pull unsuccessful. Trying again in a few seconds."
|
||||
sleep $(shuf -i 10-30 -n1)
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$gitterm" -ne 0 ]; then
|
||||
debug "Three git pulls failed, hard resetting and repulling"
|
||||
git -C "$basedir" reset --hard HEAD~50 >"$GITchanges" 2>"$GITerrors"
|
||||
git -C "$basedir" pull >>"$GITchanges" 2>>"$GITerrors"
|
||||
gitterm="$?"
|
||||
fi
|
||||
if [ "$gitterm" -ne 0 ]; then
|
||||
debug "Three git pulls failed, hard resetting and repulling"
|
||||
git -C "$basedir" reset --hard HEAD~50 >"$GITchanges" 2>"$GITerrors"
|
||||
git -C "$basedir" pull >>"$GITchanges" 2>>"$GITerrors"
|
||||
gitterm="$?"
|
||||
fi
|
||||
|
||||
if [ "$gitterm" -ne 0 ]; then
|
||||
die "GIT reported the following problem:\n$(cat "$GITerrors")"
|
||||
fi
|
||||
if [ "$gitterm" -ne 0 ]; then
|
||||
die "GIT reported the following problem:\n$(cat "$GITerrors")"
|
||||
fi
|
||||
|
||||
if egrep '^Already up[ -]to[ -]date' "$GITchanges"; then
|
||||
debug "No changes to GIT:\n$(cat "$GITchanges")"
|
||||
# Exit status should only be 0 if there was a successful build.
|
||||
# So set it to 1 here.
|
||||
exit 1
|
||||
fi
|
||||
if egrep '^Already up[ -]to[ -]date' "$GITchanges"; then
|
||||
debug "No changes to GIT:\n$(cat "$GITchanges")"
|
||||
# Exit status should only be 0 if there was a successful build.
|
||||
# So set it to 1 here.
|
||||
exit 1
|
||||
fi
|
||||
|
||||
logstatus GITlatest < "$GITchanges"
|
||||
buildrun
|
||||
logstatus GITlatest <"$GITchanges"
|
||||
buildrun
|
||||
}
|
||||
|
||||
# Clean up everything and then do an actual (full) build
|
||||
build_into(){
|
||||
build_into() {
|
||||
|
||||
# Clean up source directory.
|
||||
git -C "${basedir}" clean -dxf --exclude=status.fsfe.org/translations/data
|
||||
# Clean up source directory.
|
||||
git -C "${basedir}" clean -dxf --exclude=status.fsfe.org/translations/data
|
||||
|
||||
# Remove old stage directory.
|
||||
rm -rf "${stagedir}"
|
||||
# Remove old stage directory.
|
||||
rm -rf "${stagedir}"
|
||||
|
||||
buildrun
|
||||
buildrun
|
||||
}
|
||||
|
||||
@ -2,15 +2,15 @@
|
||||
|
||||
inc_filenames=true
|
||||
|
||||
get_language(){
|
||||
# extract language indicator from a given file name
|
||||
echo "$(echo "$1" |sed -r 's:^.*\.([a-z]{2})\.[^\.]+$:\1:')";
|
||||
get_language() {
|
||||
# extract language indicator from a given file name
|
||||
echo "$(echo "$1" | sed -r 's:^.*\.([a-z]{2})\.[^\.]+$:\1:')"
|
||||
}
|
||||
|
||||
get_shortname(){
|
||||
# get shortened version of a given file name
|
||||
# required for internal processing
|
||||
get_shortname() {
|
||||
# get shortened version of a given file name
|
||||
# required for internal processing
|
||||
|
||||
#echo "$(echo "$1" | sed -r 's:\.[a-z]{2}.xhtml$::')";
|
||||
echo "${1%.??.xhtml}"
|
||||
#echo "$(echo "$1" | sed -r 's:\.[a-z]{2}.xhtml$::')";
|
||||
echo "${1%.??.xhtml}"
|
||||
}
|
||||
|
||||
@ -2,42 +2,42 @@
|
||||
|
||||
inc_logging=true
|
||||
|
||||
logname(){
|
||||
name="$1"
|
||||
logname() {
|
||||
name="$1"
|
||||
|
||||
if [ -w "$statusdir" ] && touch "$statusdir/$name"; then
|
||||
echo "$statusdir/$name"
|
||||
elif echo "$forcedlog" |egrep -q "^${name}=.+"; then
|
||||
echo "$forcedlog" \
|
||||
| sed -rn "s;^${name}=;;p"
|
||||
else
|
||||
echo /dev/null
|
||||
fi
|
||||
if [ -w "$statusdir" ] && touch "$statusdir/$name"; then
|
||||
echo "$statusdir/$name"
|
||||
elif echo "$forcedlog" | egrep -q "^${name}=.+"; then
|
||||
echo "$forcedlog" |
|
||||
sed -rn "s;^${name}=;;p"
|
||||
else
|
||||
echo /dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
forcelog(){
|
||||
name="$1"
|
||||
forcelog() {
|
||||
name="$1"
|
||||
|
||||
[ "$(logname "$name")" = "/dev/null" ] \
|
||||
&& forcedlog="$forcedlog\n${name}=$(mktemp -t w3bldXXXXXXXXX --suffix $$)"
|
||||
[ "$(logname "$name")" = "/dev/null" ] &&
|
||||
forcedlog="$forcedlog\n${name}=$(mktemp -t w3bldXXXXXXXXX --suffix $$)"
|
||||
}
|
||||
|
||||
[ -z "$USER" ] && USER="$(whoami)"
|
||||
trap "trap - 0 2 3 6 9 15; find \"${TMPDIR:-/tmp}/\" -maxdepth 1 -user \"$USER\" -name \"w3bld*$$\" -delete" 0 2 3 6 9 15
|
||||
|
||||
logstatus(){
|
||||
# pipeline atom to write data streams into a log file
|
||||
tee "$(logname "$1")"
|
||||
logstatus() {
|
||||
# pipeline atom to write data streams into a log file
|
||||
tee "$(logname "$1")"
|
||||
}
|
||||
|
||||
t_logstatus(){
|
||||
# pipeline atom to write data streams into a log file
|
||||
while read line; do
|
||||
printf "[$(date +%T)] %s\n" "$line"
|
||||
done |logstatus "$@"
|
||||
t_logstatus() {
|
||||
# pipeline atom to write data streams into a log file
|
||||
while read line; do
|
||||
printf "[$(date +%T)] %s\n" "$line"
|
||||
done | logstatus "$@"
|
||||
}
|
||||
|
||||
logappend(){
|
||||
# pipeline atom to write data streams into a log file
|
||||
tee -a "$(logname "$1")"
|
||||
logappend() {
|
||||
# pipeline atom to write data streams into a log file
|
||||
tee -a "$(logname "$1")"
|
||||
}
|
||||
|
||||
@ -2,13 +2,12 @@
|
||||
|
||||
inc_makerules=true
|
||||
|
||||
tree_maker(){
|
||||
# walk through file tree and issue Make rules according to file type
|
||||
input="$(realpath "$1")"
|
||||
output="$(realpath "$2")"
|
||||
languages="$(ls -xw0 "${basedir}/global/languages")"
|
||||
tree_maker() {
|
||||
# walk through file tree and issue Make rules according to file type
|
||||
input="$(realpath "$1")"
|
||||
output="$(realpath "$2")"
|
||||
|
||||
cat <<EOF
|
||||
cat <<EOF
|
||||
# -----------------------------------------------------------------------------
|
||||
# Makefile for FSFE website build, phase 2
|
||||
# -----------------------------------------------------------------------------
|
||||
@ -17,7 +16,7 @@ tree_maker(){
|
||||
.DELETE_ON_ERROR:
|
||||
.SECONDEXPANSION:
|
||||
PROCESSOR = "$basedir/build/process_file.sh"
|
||||
PROCFLAGS = --build-env "${build_env:-development}" --source "$basedir" --domain "$domain"
|
||||
PROCFLAGS = --build-env "${build_env}" --source "$basedir"
|
||||
INPUTDIR = $input
|
||||
OUTPUTDIR = $output
|
||||
STATUSDIR = $statusdir
|
||||
@ -55,15 +54,15 @@ XSL_DEP = \$(firstword \$(wildcard \$(INPUTDIR)/\$*.xsl) \$(INPUTDIR)/\$(dir \$*
|
||||
all: \$(HTML_DST_FILES)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat<<EOF
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(filter %.${lang}.html,\$(HTML_DST_FILES)): \$(OUTPUTDIR)/%.${lang}.html: \$(INPUTDIR)/%.*.xhtml \$\$(XMLLIST_DEP) \$\$(XSL_DEP) \$(INPUTDIR)/global/data/texts/.texts.${lang}.xml \$(INPUTDIR)/global/data/texts/texts.en.xml \$(INPUTDIR)/global/data/topbanner/.topbanner.${lang}.xml
|
||||
echo "* Building \$*.${lang}.html"
|
||||
\${PROCESSOR} \${PROCFLAGS} process_file "\$(INPUTDIR)/\$*.${lang}.xhtml" > "\$@"
|
||||
EOF
|
||||
done
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create index.* symlinks
|
||||
@ -90,15 +89,15 @@ INDEX_DST_LINKS := \$(foreach base,\$(INDEX_DST_DIRS),\$(foreach lang,\$(LANGUAG
|
||||
all: \$(INDEX_DST_LINKS)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat<<EOF
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(filter %/index.${lang}.html,\$(INDEX_DST_LINKS)): \$(OUTPUTDIR)/%/index.${lang}.html:
|
||||
echo "* Creating symlink \$*/index.${lang}.html"
|
||||
ln -sf "\$(notdir \$*).${lang}.html" "\$@"
|
||||
EOF
|
||||
done
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Create symlinks from file.<lang>.html to file.html.<lang>
|
||||
@ -110,15 +109,15 @@ HTML_DST_LINKS := \$(foreach base,\$(HTML_DST_BASES) \$(addsuffix index,\$(INDEX
|
||||
all: \$(HTML_DST_LINKS)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat<<EOF
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(OUTPUTDIR)/%.html.${lang}:
|
||||
echo "* Creating symlink \$*.html.${lang}"
|
||||
ln -sf "\$(notdir \$*).${lang}.html" "\$@"
|
||||
EOF
|
||||
done
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Build .rss files from .xhtml sources
|
||||
@ -142,15 +141,15 @@ RSS_DST_FILES := \$(foreach base,\$(RSS_DST_BASES),\$(foreach lang,\$(LANGUAGES)
|
||||
all: \$(RSS_DST_FILES)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat<<EOF
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(OUTPUTDIR)/%.${lang}.rss: \$(INPUTDIR)/%.*.xhtml \$\$(XMLLIST_DEP) \$(INPUTDIR)/%.rss.xsl \$(INPUTDIR)/global/data/texts/.texts.${lang}.xml \$(INPUTDIR)/global/data/texts/texts.en.xml
|
||||
echo "* Building \$*.${lang}.rss"
|
||||
\${PROCESSOR} \${PROCFLAGS} process_file "\$(INPUTDIR)/\$*.${lang}.xhtml" "\$(INPUTDIR)/\$*.rss.xsl" > "\$@"
|
||||
EOF
|
||||
done
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Build .ics files from .xhtml sources
|
||||
@ -174,15 +173,15 @@ ICS_DST_FILES := \$(foreach base,\$(ICS_DST_BASES),\$(foreach lang,\$(LANGUAGES)
|
||||
all: \$(ICS_DST_FILES)
|
||||
EOF
|
||||
|
||||
for lang in ${languages}; do
|
||||
cat<<EOF
|
||||
for lang in ${languages}; do
|
||||
cat <<EOF
|
||||
\$(OUTPUTDIR)/%.${lang}.ics: \$(INPUTDIR)/%.*.xhtml \$\$(XMLLIST_DEP) \$(INPUTDIR)/%.ics.xsl \$(INPUTDIR)/global/data/texts/.texts.${lang}.xml \$(INPUTDIR)/global/data/texts/texts.en.xml
|
||||
echo "* Building \$*.${lang}.ics"
|
||||
\${PROCESSOR} \${PROCFLAGS} process_file "\$(INPUTDIR)/\$*.${lang}.xhtml" "\$(INPUTDIR)/\$*.ics.xsl" > "\$@"
|
||||
EOF
|
||||
done
|
||||
done
|
||||
|
||||
cat <<EOF
|
||||
cat <<EOF
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Copy images, docments etc
|
||||
|
||||
@ -3,21 +3,21 @@
|
||||
inc_misc=true
|
||||
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
|
||||
|
||||
debug(){
|
||||
if [ "$#" -ge 1 ]; then
|
||||
echo "$(date '+%F %T'): $@" |logappend debug >&2
|
||||
else
|
||||
logappend debug >&2
|
||||
fi
|
||||
debug() {
|
||||
if [ "$#" -ge 1 ]; then
|
||||
echo "$(date '+%F %T'): $@" | logappend debug >&2
|
||||
else
|
||||
logappend debug >&2
|
||||
fi
|
||||
}
|
||||
|
||||
print_error(){
|
||||
echo "Error - $@" |logappend lasterror >&2
|
||||
echo "Run '$0 --help' to see usage instructions" >&2
|
||||
print_error() {
|
||||
echo "Error - $@" | logappend lasterror >&2
|
||||
echo "Run '$0 --help' to see usage instructions" >&2
|
||||
}
|
||||
|
||||
die(){
|
||||
echo "$(date '+%F %T'): Fatal - $@" |logappend lasterror >&2
|
||||
date +%s |logstatus end_time
|
||||
exit 1
|
||||
die() {
|
||||
echo "$(date '+%F %T'): Fatal - $@" | logappend lasterror >&2
|
||||
date +%s | logstatus end_time
|
||||
exit 1
|
||||
}
|
||||
|
||||
@ -6,6 +6,6 @@ basedir="${0%/*}/.."
|
||||
. "$basedir/build/arguments.sh"
|
||||
|
||||
case "$command" in
|
||||
process_file) process_file "$workfile" "$processor" ;;
|
||||
*) die "Unrecognised command or no command given" ;;
|
||||
process_file) process_file "$workfile" "$processor" ;;
|
||||
*) die "Unrecognised command or no command given" ;;
|
||||
esac
|
||||
|
||||
@ -2,48 +2,46 @@
|
||||
|
||||
inc_processor=true
|
||||
[ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh"
|
||||
[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh"
|
||||
[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh"
|
||||
|
||||
process_file(){
|
||||
infile="$1"
|
||||
processor="$2"
|
||||
process_file() {
|
||||
infile="$1"
|
||||
processor="$2"
|
||||
|
||||
shortname=$(get_shortname "$infile")
|
||||
lang=$(get_language "$infile")
|
||||
shortname=$(get_shortname "$infile")
|
||||
lang=$(get_language "$infile")
|
||||
|
||||
if [ -z "${processor}" ]; then
|
||||
if [ -f "${shortname}.xsl" ]; then
|
||||
processor="${shortname}.xsl"
|
||||
else
|
||||
# Actually use the symlink target, so the relative includes are searched
|
||||
# in the correct directory.
|
||||
processor="$(realpath "${shortname%/*}/.default.xsl")"
|
||||
fi
|
||||
fi
|
||||
if [ -z "${processor}" ]; then
|
||||
if [ -f "${shortname}.xsl" ]; then
|
||||
processor="${shortname}.xsl"
|
||||
else
|
||||
# Actually use the symlink target, so the relative includes are searched
|
||||
# in the correct directory.
|
||||
processor="$(realpath "${shortname%/*}/.default.xsl")"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Make sure that the following pipe exits with a nonzero exit code if *any*
|
||||
# of the commands fails.
|
||||
set -o pipefail
|
||||
# Make sure that the following pipe exits with a nonzero exit code if *any*
|
||||
# of the commands fails.
|
||||
set -o pipefail
|
||||
|
||||
# The sed command of death below does the following:
|
||||
# 1. Remove https://fsfe.org (or https://test.fsfe.org) from the start of all
|
||||
# links
|
||||
# 2. Change links from /foo/bar.html into /foo/bar.xx.html
|
||||
# 3. Change links from foo/bar.html into foo/bar.xx.html
|
||||
# 4. Same for .rss and .ics links
|
||||
# 5. Change links from /foo/bar/ into /foo/bar/index.xx.html
|
||||
# 6. Change links from foo/bar/ into foo/bar/index.xx.html
|
||||
# ... where xx is the language code.
|
||||
# Everything is duplicated to allow for the href attribute to be enclosed in
|
||||
# single or double quotes.
|
||||
# I am strongly convinced that there must be a less obfuscated way of doing
|
||||
# this. --Reinhard
|
||||
# The sed command of death below does the following:
|
||||
# 1. Remove https://fsfe.org (or https://test.fsfe.org) from the start of all
|
||||
# links
|
||||
# 2. Change links from /foo/bar.html into /foo/bar.xx.html
|
||||
# 3. Change links from foo/bar.html into foo/bar.xx.html
|
||||
# 4. Same for .rss and .ics links
|
||||
# 5. Change links from /foo/bar/ into /foo/bar/index.xx.html
|
||||
# 6. Change links from foo/bar/ into foo/bar/index.xx.html
|
||||
# ... where xx is the language code.
|
||||
# Everything is duplicated to allow for the href attribute to be enclosed in
|
||||
# single or double quotes.
|
||||
# I am strongly convinced that there must be a less obfuscated way of doing
|
||||
# this. --Reinhard
|
||||
|
||||
build_xmlstream "$shortname" "$lang" \
|
||||
| xsltproc --stringparam "build-env" "${build_env:-development}" "$processor" - \
|
||||
| sed -r ':X; N; $!bX;
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="(https?:)?//'"$domain"'/([^"]*)";<\1\2 href="/\4";gI
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href='\''(https?:)?//'"$domain"'/([^'\'']*)'\'';<\1\2 href='\''/\4'\'';gI
|
||||
build_xmlstream "$shortname" "$lang" |
|
||||
xsltproc --stringparam "build-env" "${build_env}" "$processor" - |
|
||||
sed -r ':X; N; $!bX;
|
||||
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="((https?:)?//[^"]*)";<\1\2 href="#== norewrite ==\3";gI
|
||||
s;<[\r\n\t ]*(a|link)([\r\n\t ][^>]*)?[\r\n\t ]href="([^#"])([^"]*/)?([^\./"]*\.)(html|rss|ics)(#[^"]*)?";<\1\2 href="\3\4\5'"$lang"'.\6\7";gI
|
||||
|
||||
@ -2,124 +2,124 @@
|
||||
|
||||
inc_scaffold=true
|
||||
|
||||
get_version(){
|
||||
version=$(xsltproc $basedir/build/xslt/get_version.xsl $1)
|
||||
echo ${version:-0}
|
||||
get_version() {
|
||||
version=$(xsltproc $basedir/build/xslt/get_version.xsl $1)
|
||||
echo ${version:-0}
|
||||
}
|
||||
|
||||
include_xml(){
|
||||
# include second level elements of a given XML file
|
||||
# this emulates the behaviour of the original
|
||||
# build script which wasn't able to load top
|
||||
# level elements from any file
|
||||
if [ -f "$1" ]; then
|
||||
# Remove <version> because the filename attribute would otherwise be added
|
||||
# to this element instead of the actual content element.
|
||||
sed 's;<version>.*</version>;;' "$1" | \
|
||||
sed -r ':X; $bY; N; bX; :Y;
|
||||
include_xml() {
|
||||
# include second level elements of a given XML file
|
||||
# this emulates the behaviour of the original
|
||||
# build script which wasn't able to load top
|
||||
# level elements from any file
|
||||
if [ -f "$1" ]; then
|
||||
# Remove <version> because the filename attribute would otherwise be added
|
||||
# to this element instead of the actual content element.
|
||||
sed 's;<version>.*</version>;;' "$1" |
|
||||
sed -r ':X; $bY; N; bX; :Y;
|
||||
s:<(\?[xX][mM][lL]|!DOCTYPE)[[:space:]]+[^>]+>::g
|
||||
s:<[^!][^>]*>::;
|
||||
s:</[^>]*>([^<]*((<[^>]+/>|<!([^>]|<[^>]*>)*>|<\?[^>]+>)[^<]*)*)?$:\1:;'
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
get_attributes(){
|
||||
# get attributes of top level element in a given
|
||||
# XHTML file
|
||||
sed -rn ':X; N; $!bX;
|
||||
get_attributes() {
|
||||
# get attributes of top level element in a given
|
||||
# XHTML file
|
||||
sed -rn ':X; N; $!bX;
|
||||
s;^.*<[\n\t\r ]*([xX]|[xX]?[hH][tT])[mM][lL][\n\t\r ]+([^>]*)>.*$;\2;p' "$1"
|
||||
}
|
||||
|
||||
list_langs(){
|
||||
# list all languages a file exists in by globbing up
|
||||
# the shortname (i.e. file path with file ending omitted)
|
||||
# output is readily formatted for inclusion
|
||||
# in xml stream
|
||||
for file in "${1}".[a-z][a-z].xhtml; do
|
||||
language="${file: -8:2}"
|
||||
text="$(echo -n $(cat "${basedir}/global/languages/${language}"))"
|
||||
echo "<tr id=\"${language}\">${text}</tr>"
|
||||
done
|
||||
list_langs() {
|
||||
# list all languages a file exists in by globbing up
|
||||
# the shortname (i.e. file path with file ending omitted)
|
||||
# output is readily formatted for inclusion
|
||||
# in xml stream
|
||||
for file in "${1}".[a-z][a-z].xhtml; do
|
||||
language="${file: -8:2}"
|
||||
text="$(echo -n $(cat "${basedir}/global/languages/${language}"))"
|
||||
echo "<tr id=\"${language}\">${text}</tr>"
|
||||
done
|
||||
}
|
||||
|
||||
auto_sources(){
|
||||
# import elements from source files, add file name
|
||||
# attribute to first element included from each file
|
||||
shortname="$1"
|
||||
lang="$2"
|
||||
auto_sources() {
|
||||
# import elements from source files, add file name
|
||||
# attribute to first element included from each file
|
||||
shortname="$1"
|
||||
lang="$2"
|
||||
|
||||
list_file="$(dirname ${shortname})/.$(basename ${shortname}).xmllist"
|
||||
list_file="$(dirname ${shortname})/.$(basename ${shortname}).xmllist"
|
||||
|
||||
if [ -f "${list_file}" ]; then
|
||||
cat "${list_file}" | while read path; do
|
||||
base="$(basename ${path})"
|
||||
if [ -f "${basedir}/${path}.${lang}.xml" ]; then
|
||||
printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.${lang}.xml")"
|
||||
elif [ -f "${basedir}/${path}.en.xml" ]; then
|
||||
printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.en.xml")"
|
||||
fi
|
||||
done \
|
||||
| sed -r ':X; N; $!bX;
|
||||
if [ -f "${list_file}" ]; then
|
||||
cat "${list_file}" | while read path; do
|
||||
base="$(basename ${path})"
|
||||
if [ -f "${basedir}/${path}.${lang}.xml" ]; then
|
||||
printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.${lang}.xml")"
|
||||
elif [ -f "${basedir}/${path}.en.xml" ]; then
|
||||
printf '\n### filename="%s" ###\n%s' "${base#.}" "$(include_xml "${basedir}/${path}.en.xml")"
|
||||
fi
|
||||
done |
|
||||
sed -r ':X; N; $!bX;
|
||||
s;\n### (filename="[^\n"]+") ###\n[^<]*(<![^>]+>[^<]*)*(<([^/>]+/)*([^/>]+))(/?>);\2\3 \1\6;g;'
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
build_xmlstream(){
|
||||
# assemble the xml stream for feeding into xsltproc
|
||||
# the expected shortname and language flag indicate
|
||||
# a single xhtml page to be built
|
||||
shortname="$1"
|
||||
lang="$2"
|
||||
build_xmlstream() {
|
||||
# assemble the xml stream for feeding into xsltproc
|
||||
# the expected shortname and language flag indicate
|
||||
# a single xhtml page to be built
|
||||
shortname="$1"
|
||||
lang="$2"
|
||||
|
||||
olang="$(echo "${shortname}".[a-z][a-z].xhtml "${shortname}".[e]n.xhtml |sed -rn 's;^.*\.([a-z]{2})\.xhtml.*$;\1;p')"
|
||||
dirname="${shortname%/*}/"
|
||||
topbanner_xml="$basedir/global/data/topbanner/.topbanner.${lang}.xml"
|
||||
texts_xml="$basedir/global/data/texts/.texts.${lang}.xml"
|
||||
date="$(date +%Y-%m-%d)"
|
||||
time="$(date +%H:%M:%S)"
|
||||
|
||||
if [ -f "${shortname}.${lang}.xhtml" ]; then
|
||||
act_lang="$lang"
|
||||
translation_state="up-to-date"
|
||||
[ $(get_version "${shortname}.${olang}.xhtml") -gt $(get_version "${shortname}.${lang}.xhtml") ] && translation_state="outdated"
|
||||
[ $(( $(get_version "${shortname}.${olang}.xhtml") - 3 )) -gt $(get_version "${shortname}.${lang}.xhtml") ] && act_lang="$olang" && translation_state="very-outdated"
|
||||
else
|
||||
act_lang="$olang"
|
||||
translation_state="untranslated"
|
||||
fi
|
||||
olang="$(echo "${shortname}".[a-z][a-z].xhtml "${shortname}".[e]n.xhtml | sed -rn 's;^.*\.([a-z]{2})\.xhtml.*$;\1;p')"
|
||||
dirname="${shortname%/*}/"
|
||||
topbanner_xml="$basedir/global/data/topbanner/.topbanner.${lang}.xml"
|
||||
texts_xml="$basedir/global/data/texts/.texts.${lang}.xml"
|
||||
date="$(date +%Y-%m-%d)"
|
||||
time="$(date +%H:%M:%S)"
|
||||
|
||||
infile="${shortname}.${act_lang}.xhtml"
|
||||
if [ -f "${shortname}.${lang}.xhtml" ]; then
|
||||
act_lang="$lang"
|
||||
translation_state="up-to-date"
|
||||
[ $(get_version "${shortname}.${olang}.xhtml") -gt $(get_version "${shortname}.${lang}.xhtml") ] && translation_state="outdated"
|
||||
[ $(($(get_version "${shortname}.${olang}.xhtml") - 3)) -gt $(get_version "${shortname}.${lang}.xhtml") ] && act_lang="$olang" && translation_state="very-outdated"
|
||||
else
|
||||
act_lang="$olang"
|
||||
translation_state="untranslated"
|
||||
fi
|
||||
|
||||
cat <<-EOF
|
||||
<buildinfo
|
||||
date="$date"
|
||||
original="$olang"
|
||||
filename="/${shortname#"$basedir"/}"
|
||||
fileurl="/${shortname#"$basedir"/*/}"
|
||||
dirname="/${dirname#"$basedir"/}"
|
||||
language="$lang"
|
||||
translation_state="$translation_state"
|
||||
>
|
||||
infile="${shortname}.${act_lang}.xhtml"
|
||||
|
||||
<trlist>
|
||||
$(list_langs "$shortname")
|
||||
</trlist>
|
||||
cat <<-EOF
|
||||
<buildinfo
|
||||
date="$date"
|
||||
original="$olang"
|
||||
filename="/${shortname#"$basedir"/}"
|
||||
fileurl="/${shortname#"$basedir"/*/}"
|
||||
dirname="/${dirname#"$basedir"/}"
|
||||
language="$lang"
|
||||
translation_state="$translation_state"
|
||||
>
|
||||
|
||||
<topbanner>$(include_xml "$topbanner_xml")</topbanner>
|
||||
<textsetbackup>$(include_xml "$basedir/global/data/texts/texts.en.xml")</textsetbackup>
|
||||
<textset>$(include_xml "$texts_xml")</textset>
|
||||
|
||||
<document
|
||||
language="$act_lang"
|
||||
$(get_attributes "$infile")
|
||||
>
|
||||
<set>
|
||||
$(auto_sources "${shortname}" "$lang")
|
||||
</set>
|
||||
<trlist>
|
||||
$(list_langs "$shortname")
|
||||
</trlist>
|
||||
|
||||
$(include_xml "$infile")
|
||||
</document>
|
||||
<topbanner>$(include_xml "$topbanner_xml")</topbanner>
|
||||
<textsetbackup>$(include_xml "$basedir/global/data/texts/texts.en.xml")</textsetbackup>
|
||||
<textset>$(include_xml "$texts_xml")</textset>
|
||||
|
||||
</buildinfo>
|
||||
<document
|
||||
language="$act_lang"
|
||||
$(get_attributes "$infile")
|
||||
>
|
||||
<set>
|
||||
$(auto_sources "${shortname}" "$lang")
|
||||
</set>
|
||||
|
||||
$(include_xml "$infile")
|
||||
</document>
|
||||
|
||||
</buildinfo>
|
||||
EOF
|
||||
}
|
||||
|
||||
@ -19,7 +19,7 @@ ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS))
|
||||
ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS))
|
||||
|
||||
# Languages in which the template exists
|
||||
ARCH_LANGS := $(suffix $(basename $(wildcard archive-template.??.xhtml)))
|
||||
ARCH_LANGS := $(filter $(foreach lang,$(languages),.$(lang)),$(suffix $(basename $(wildcard archive-template.??.xhtml))))
|
||||
|
||||
# .xhtml files to generate
|
||||
ARCH_XHTML := $(foreach year,$(ARCH_YEARS),$(foreach lang,$(ARCH_LANGS),$(year)/index$(lang).xhtml))
|
||||
|
||||
@ -19,7 +19,7 @@ ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS))
|
||||
ARCH_YEARS := $(filter-out $(lastword $(ARCH_YEARS)),$(ARCH_YEARS))
|
||||
|
||||
# Languages in which the template exists
|
||||
ARCH_LANGS := $(suffix $(basename $(wildcard archive-template.??.xhtml)))
|
||||
ARCH_LANGS := $(filter $(foreach lang,$(languages),.$(lang)),$(suffix $(basename $(wildcard archive-template.??.xhtml))))
|
||||
|
||||
# .xhtml files to generate
|
||||
ARCH_XHTML := $(foreach year,$(ARCH_YEARS),$(foreach lang,$(ARCH_LANGS),$(year)/index$(lang).xhtml))
|
||||
@ -62,7 +62,7 @@ all: $(GENERATED_XML_SOURCES)
|
||||
# -----------------------------------------------------------------------------
|
||||
|
||||
# All existing .xhtml files
|
||||
XHTML := $(shell ls */*.??.xhtml | xargs grep -l '<html newsdate')
|
||||
XHTML := $(filter $(foreach lang,$(languages),%.$(lang).xhtml),$(shell ls */*.??.xhtml | xargs grep -l '<html newsdate'))
|
||||
|
||||
# List of .xml files to generate
|
||||
XML := $(patsubst %.xhtml,%.xml,$(subst /,/.,$(XHTML)))
|
||||
|
||||
@ -40,11 +40,7 @@ for dir in ${!files_by_dir[@]}; do
|
||||
if [ "${file}" -nt "${dir}/.localmenu.en.xml" ]; then
|
||||
|
||||
# Find out which languages to generate.
|
||||
languages=$(
|
||||
ls ${files_by_dir[${dir}]} \
|
||||
| sed 's/.*\.\(..\)\.xhtml/\1/' \
|
||||
| sort --uniq
|
||||
)
|
||||
languages="$1"
|
||||
|
||||
# Compile the list of base filenames of the source files.
|
||||
basefiles=$(
|
||||
|
||||
@ -29,6 +29,7 @@ set -e
|
||||