Remove some spaghetti from build script
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Reinhard Müller 2020-04-24 08:52:01 +02:00
parent ccde340ea6
commit 785d03c6b9
15 changed files with 120 additions and 180 deletions

View File

@ -68,7 +68,7 @@ $(SUBDIRS): .FORCE
# otherwise. This symlinks make sure that phase 2 can easily use the right file
# for each language, also as a prerequisite in the Makefile.
LANGUAGES := $(shell . build/languages.sh && get_languages)
LANGUAGES := $(shell . build/languages.sh && echo `languages | cut -d\ -f1`)
TEXTS_LINKS := $(foreach lang,$(LANGUAGES),tools/.texts-$(lang).xml)

View File

@ -78,7 +78,7 @@ if [ -z "$inc_arguments" ]; then
process_file) [ -z "$workfile" ] && die "Need at least input file" ;;
build_xmlstream) [ -z "$workfile" ] && die "Missing xhtml file name" ;;
tree_maker) [ -z "$target" ] && die "Missing target location" ;;
*help*) print_help; exit 0 ;;
*help*) cat "$basedir/build/HELP"; exit 0 ;;
*) die "Urecognised command or no command given" ;;
esac
fi

View File

@ -61,12 +61,9 @@ fi
[ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh"
[ -z "$inc_buildrun" ] && . "$basedir/build/buildrun.sh"
[ -z "$inc_languages" ] && . "$basedir/build/languages.sh"
[ -z "$inc_makerules" ] && . "$basedir/build/makerules.sh"
[ -z "$inc_processor" ] && . "$basedir/build/processor.sh"
[ -z "$inc_scaffold" ] && . "$basedir/build/scaffold.sh"
[ -z "$inc_sources" ] && . "$basedir/build/sources.sh"
[ -z "$inc_stirrups" ] && . "$basedir/build/stirrups.sh"
case "$command" in
git_build_into) if [ "${statusdir}/full_build" -nt "${statusdir}/index.cgi" ]; then

View File

@ -2,10 +2,33 @@
inc_buildrun=true
[ -z "$inc_makerules" ] && . "$basedir/build/makerules.sh"
[ -z "$inc_stirrups" ] && . "$basedir/build/stirrups.sh"
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
match(){
printf %s "$1" | egrep -q "$2"
}
dir_maker(){
# set up directory tree for output
# optimise by only issuing mkdir commands
# for leaf directories
input="${1%/}"
output="${2%/}"
curpath="$output"
find "$input" -depth -type d \
\! -path '*/.svn' \! -path '*/.svn/*' \
\! -path '*/.git' \! -path '*/.git/*' \
-printf '%P\n' \
| while read filepath; do
oldpath="$curpath"
curpath="$output/$filepath/"
srcdir="$output/source/$filepath/"
match "$oldpath" "^$curpath" || mkdir -p "$curpath" "$srcdir"
done
}
# The actual build
buildrun(){
set -o pipefail

View File

@ -1,22 +1,6 @@
#!/usr/bin/env bash
inc_filenames=true
[ -z "$inc_languages" ] && . "$basedir/build/languages.sh"
list_langs(){
# list all languages a file exists in by globbing up
# the shortname (i.e. file path with file ending omitted)
# output is readily formatted for inclusion
# in xml stream
shortname="$1"
langfilter=$(
echo "$shortname".[a-z][a-z].xhtml \
| sed -r 's;[^ ]+.([a-z]{2}).xhtml;\1;g;s; ;|;g'
)
languages |egrep "^($langfilter) " \
| sed -r 's:^([a-z]{2}) (.+)$:<tr id="\1">\2</tr>:g'
}
get_language(){
# extract language indicator from a given file name
@ -30,21 +14,3 @@ get_shortname(){
#echo "$(echo "$1" | sed -r 's:\.[a-z]{2}.xhtml$::')";
echo "${1%.??.xhtml}"
}
get_processor(){
# find the xslt script which is responsible for processing
# a given xhtml file.
# expects the shortname of the file as input (i.e. the
# the file path without language and file endings)
shortname="$1"
if [ -f "${shortname}.xsl" ]; then
echo "${shortname}.xsl"
else
location="${shortname%/*}"
until [ -f "$location/default.xsl" -o "$location" = . -o "$location" = / ]; do
location="${location%/*}"
done
echo "$location/default.xsl"
fi
}

View File

@ -1,17 +0,0 @@
#!/usr/bin/env bash
inc_fundraising=true
[ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh"
[ -z "$inc_languages" ] && . "$basedir/build/languages.sh"
get_fundraisingfile(){
# get the fundraising file for a given language
# TODO: integrate with regular texts function
lang="$1"
if [ -f "$basedir/fundraising-${lang}.xml" ]; then
echo "$basedir/fundraising-${lang}.xml"
elif [ -f "$basedir/fundraising-en.xml" ]; then
echo "$basedir/fundraising-en.xml"
fi
}

View File

@ -39,8 +39,3 @@ uk Українська
zh 漢語
EOL
}
get_languages(){
# The list of all languages in one line, separated by blanks
echo `languages | cut -d\ -f1`
}

View File

@ -7,7 +7,7 @@ tree_maker(){
# walk through file tree and issue Make rules according to file type
input="$(realpath "$1")"
output="$(realpath "$2")"
languages=$(get_languages)
languages=$(echo $(languages | cut -d\ -f1))
cat <<EOF
# -----------------------------------------------------------------------------
@ -18,7 +18,7 @@ tree_maker(){
.DELETE_ON_ERROR:
.SECONDEXPANSION:
PROCESSOR = "$basedir/build/process_file.sh"
PROCFLAGS = --build-env "${build_env:-development}" --source "$basedir" --statusdir "$statusdir" --domain "$domain"
PROCFLAGS = --build-env "${build_env:-development}" --source "$basedir" --domain "$domain"
INPUTDIR = $input
OUTPUTDIR = $output
STATUSDIR = $statusdir

View File

@ -3,14 +3,6 @@
inc_misc=true
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
print_help(){
cat "$basedir/build/HELP"
}
match(){
printf %s "$1" |egrep -q "$2"
}
debug(){
if [ "$#" -ge 1 ]; then
echo "$(date '+%F %T'): $@" |logappend debug >&2

View File

@ -10,12 +10,35 @@ process_file(){
shortname=$(get_shortname "$infile")
lang=$(get_language "$infile")
[ -z "$processor" ] && processor="$(get_processor "$shortname")"
if [ -z "${processor}" ]; then
if [ -f "${shortname}.xsl" ]; then
processor="${shortname}.xsl"
else
# Actually use the symlink target, so the relative includes are searched
# in the correct directory.
processor="$(realpath "${shortname%/*}/.default.xsl")"
fi
fi
# Make sure that the following pipe exits with a nonzero exit code if *any*
# of the commands fails.
set -o pipefail
# The sed command of death below does the following:
# 1. Remove https://fsfe.org (or https://test.fsfe.org) from the start of all
# links
# 2. Change links from /foo/bar.html into /foo/bar.xx.html
# 3. Change links from foo/bar.html into foo/bar.xx.html
# 4. Same for .rss and .ics links
# 5. Change links from /foo/bar/ into /foo/bar/index.xx.html
# 6. Change links from foo/bar/ into foo/bar/index.xx.html
# ... where xx is the language code.
# Everything is duplicated to allow for the href attribute to be enclosed in
# single or double quotes.
# I am strongly convinced that there must be a less obfuscated way of doing
# this. --Reinhard
build_xmlstream "$shortname" "$lang" \
| xsltproc --stringparam "build-env" "${build_env:-development}" "$processor" - \
| sed -r ':X; N; $!bX;

View File

@ -1,17 +1,78 @@
#!/usr/bin/env bash
inc_scaffold=true
[ -z "$inc_xmlfiles" ] && . "$basedir/build/xmlfiles.sh"
[ -z "$inc_translations" ] && . "$basedir/build/translations.sh"
[ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh"
[ -z "$inc_fundraising" ] && . "$basedir/build/fundraising.sh"
[ -z "$inc_sources" ] && . "$basedir/build/sources.sh"
[ -z "$inc_languages" ] && . "$basedir/build/languages.sh"
get_version(){
version=$(xsltproc $basedir/build/xslt/get_version.xsl $1)
echo ${version:-0}
}
include_xml(){
# include second level elements of a given XML file
# this emulates the behaviour of the original
# build script which wasn't able to load top
# level elements from any file
if [ -f "$1" ]; then
sed -r ':X; $bY; N; bX; :Y;
s:<(\?[xX][mM][lL]|!DOCTYPE)[[:space:]]+[^>]+>::g
s:<[^!][^>]*>::;
s:</[^>]*>([^<]*((<[^>]+/>|<!([^>]|<[^>]*>)*>|<\?[^>]+>)[^<]*)*)?$:\1:;' "$1"
fi
}
get_attributes(){
# get attributes of top level element in a given
# XHTML file
sed -rn ':X; N; $!bX;
s;^.*<[\n\t\r ]*([xX]|[xX]?[hH][tT])[mM][lL][\n\t\r ]+([^>]*)>.*$;\2;p' "$1"
}
list_langs(){
# list all languages a file exists in by globbing up
# the shortname (i.e. file path with file ending omitted)
# output is readily formatted for inclusion
# in xml stream
shortname="$1"
langfilter=$(
echo "$shortname".[a-z][a-z].xhtml \
| sed -r 's;[^ ]+.([a-z]{2}).xhtml;\1;g;s; ;|;g'
)
languages | egrep "^($langfilter) " \
| sed -r 's:^([a-z]{2}) (.+)$:<tr id="\1">\2</tr>:g'
}
list_sources(){
# read a .xmllist file and generate a list
# of all referenced xml files with preference
# for a given language
shortname="$1"
lang="$2"
list_file="`dirname ${shortname}`/.`basename ${shortname}`.xmllist"
if [ -f "${list_file}" ]; then
cat "${list_file}" | while read base; do
echo "${basedir}/${base}".[a-z][a-z].xml "${basedir}/${base}".en.[x]ml "${basedir}/${base}.${lang}".[x]ml
done | sed -rn 's;^(.* )?([^ ]+\.[a-z]{2}\.xml).*$;\2;p'
fi
}
auto_sources(){
# import elements from source files, add file name
# attribute to first element included from each file
shortname="$1"
lang="$2"
list_sources "$shortname" "$lang" \
| while read source; do
printf '\n### filename="%s" ###\n%s' "$source" "$(include_xml "$source")"
done \
| sed -r ':X; N; $!bX;
s;\n### (filename="[^\n"]+") ###\n[^<]*(<![^>]+>[^<]*)*(<([^/>]+/)*([^/>]+))(/?>);\2\3 \1\6;g;'
}
build_xmlstream(){
# assemble the xml stream for feeding into xsltproc
# the expected shortname and language flag indicate
@ -21,8 +82,8 @@ build_xmlstream(){
olang="$(echo "${shortname}".[a-z][a-z].xhtml "${shortname}".[e]n.xhtml |sed -rn 's;^.*\.([a-z]{2})\.xhtml.*$;\1;p')"
dirname="${shortname%/*}/"
texts_xml=$(get_textsfile $lang)
fundraising_xml=$(get_fundraisingfile $lang)
texts_xml="$basedir/tools/.texts-${lang}.xml"
fundraising_xml="$basedir/.fundraising.${lang}.xml"
date="$(date +%Y-%m-%d)"
time="$(date +%H:%M:%S)"
outdated=no

View File

@ -1,35 +0,0 @@
#!/usr/bin/env bash
inc_sources=true
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
[ -z "$inc_xmlfiles" ] && . "$basedir/build/xmlfiles.sh"
list_sources(){
# read a .xmllist file and generate a list
# of all referenced xml files with preference
# for a given language
shortname="$1"
lang="$2"
list_file="`dirname ${shortname}`/.`basename ${shortname}`.xmllist"
if [ -f "${list_file}" ]; then
cat "${list_file}" | while read base; do
echo "${basedir}/${base}".[a-z][a-z].xml "${basedir}/${base}".en.[x]ml "${basedir}/${base}.${lang}".[x]ml
done | sed -rn 's;^(.* )?([^ ]+\.[a-z]{2}\.xml).*$;\2;p'
fi
}
auto_sources(){
# import elements from source files, add file name
# attribute to first element included from each file
shortname="$1"
lang="$2"
list_sources "$shortname" "$lang" \
| while read source; do
printf '\n### filename="%s" ###\n%s' "$source" "$(include_xml "$source")"
done \
| sed -r ':X; N; $!bX;
s;\n### (filename="[^\n"]+") ###\n[^<]*(<![^>]+>[^<]*)*(<([^/>]+/)*([^/>]+))(/?>);\2\3 \1\6;g;'
}

View File

@ -1,24 +0,0 @@
#!/usr/bin/env bash
inc_stirrups=true
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
dir_maker(){
# set up directory tree for output
# optimise by only issuing mkdir commands
# for leaf directories
input="${1%/}"
output="${2%/}"
curpath="$output"
find "$input" -depth -type d \
\! -path '*/.svn' \! -path '*/.svn/*' \
\! -path '*/.git' \! -path '*/.git/*' \
-printf '%P\n' \
| while read filepath; do
oldpath="$curpath"
curpath="$output/$filepath/"
srcdir="$output/source/$filepath/"
match "$oldpath" "^$curpath" || mkdir -p "$curpath" "$srcdir"
done
}

View File

@ -1,17 +0,0 @@
#!/usr/bin/env bash
inc_translations=true
[ -z "$inc_filenames" ] && . "$basedir/build/filenames.sh"
[ -z "$inc_languages" ] && . "$basedir/build/languages.sh"
get_textsfile(){
# get the texts file for a given language
# fall back to English if necessary
lang="$1"
if [ -f "$basedir/tools/texts-${1}.xml" ]; then
echo "$basedir/tools/texts-${1}.xml"
else
echo "$basedir/tools/texts-en.xml"
fi
}

View File

@ -1,24 +0,0 @@
#!/usr/bin/env bash
inc_xmlfiles=true
include_xml(){
# include second level elements of a given XML file
# this emulates the behaviour of the original
# build script which wasn't able to load top
# level elements from any file
if [ -f "$1" ]; then
sed -r ':X; $bY; N; bX; :Y;
s:<(\?[xX][mM][lL]|!DOCTYPE)[[:space:]]+[^>]+>::g
s:<[^!][^>]*>::;
s:</[^>]*>([^<]*((<[^>]+/>|<!([^>]|<[^>]*>)*>|<\?[^>]+>)[^<]*)*)?$:\1:;' "$1"
fi
}
get_attributes(){
# get attributes of top level element in a given
# XHTML file
sed -rn ':X; N; $!bX;
s;^.*<[\n\t\r ]*([xX]|[xX]?[hH][tT])[mM][lL][\n\t\r ]+([^>]*)>.*$;\2;p' "$1"
}