From b5759da7a0fae0ed2ae7387590119f1efb69cf4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Reinhard=20M=C3=BCller?= Date: Mon, 11 Mar 2019 02:35:51 +0100 Subject: [PATCH] Generate list of included XML files in prebuild run --- .gitignore | 12 ++-- Makefile | 58 ++++---------------- build/arguments.sh | 17 ------ build/build_main.sh | 3 - build/buildrun.sh | 4 +- build/make_tagmaps.sh | 99 --------------------------------- build/make_xmllists.sh | 118 ++++++++++++++++++++++++++++++++++++++++ build/makerules.sh | 15 ++--- build/scaffold.sh | 2 +- build/source_globber.sh | 13 ----- build/sources.sh | 60 +++----------------- build/xslt/get_tags.xsl | 2 +- tools/tagmaps/README | 3 - 13 files changed, 153 insertions(+), 253 deletions(-) delete mode 100755 build/make_tagmaps.sh create mode 100755 build/make_xmllists.sh delete mode 100755 build/source_globber.sh delete mode 100644 tools/tagmaps/README diff --git a/.gitignore b/.gitignore index ae3718ed58..f1467848d2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ -tools/tagmaps/*.map -tags/tagged-*.sources -tags/tagged-*.en.xhtml -d_day.en.xml -d_month.en.xml -d_year.en.xml localmenuinfo.en.xml +d_year.en.xml +d_month.en.xml +d_day.en.xml +.default.xsl +.*.xmllist +tags/tagged-*.en.xhtml news/*/.*.??.xml about/printable/archive/printable.en.xml diff --git a/Makefile b/Makefile index 5975b7b31b..5fef924a2a 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,6 @@ # This Makefile creates some .xml and xhtml files which serve as source files # in the main build run. It is executed in the source directory, before the # Makefile for the main build run is constructed and executed. -# -# It also touches all the .sources files which refer to added, modified, or -# deleted .xml files. This way, we avoid that in the main build Makefile each -# .html file has to list a long and changing list of .xml prerequisites - it is -# sufficient to just have the .sources file as a prerequisite. # ----------------------------------------------------------------------------- .PHONY: all .FORCE @@ -56,49 +51,16 @@ d_year.en.xml: $(if $(findstring $(YEAR),$(shell cat d_year.en.xml)),,.FORCE) printf %s\\n '$(YEAR)' >$@ # ----------------------------------------------------------------------------- -# Generate tag maps +# Generate XML filelists # ----------------------------------------------------------------------------- -# Generation of tag maps is handled in an external script which generates -# tools/tagmaps/*.map, tags/tagged-*.en.xhtml, and tags/tagged-*.sources. The -# tag map files cannot be targets in this Makefile, because the list of map -# files is not known when the Makefile starts - some new tags might be created -# when generating the .xml files in the news/* subdirectories. +# Generation of XML filelists is handled in an external script which generates +# all .xmllist and the tags/tagged-*.en.xhtml files. These files cannot be +# targets in this Makefile, because the list of tags is not known when the +# Makefile starts - some new tags might be created when generating the .xml +# files in the news/* subdirectories. -.PHONY: tagmaps -all: tagmaps -tagmaps: $(SUBDIRS) - @build/make_tagmaps.sh - - -# ----------------------------------------------------------------------------- -# Touch .sources files for which the web pages must be rebuilt -# ----------------------------------------------------------------------------- - -# The .sources files in the tags directory are already handled by -# make_tagmaps.sh -SOURCES_FILES := $(shell find * -name '*.sources' -not -path 'tags/*') - -# Secondary expansion means that the SOURCEDIRS and SOURCEREQS variables will -# be executed once for each target, and it allows us to use the variable $@ -# within the expression. -.SECONDEXPANSION: - -# This variable contains all the directories listed in the .sources file. It is -# added to the prerequisites so that the removal of a file from such a -# directory also triggers a rebuild of the web pages which have included the -# now removed file. However, we explicitly exclude "." (the root source -# directory) because that also contains a lot of other files. -SOURCES_DIRS = $(shell ls -d `sed -rn 's;^(.*/)[^/]*:(\[.*\])$$;\1;gp' $@` | grep -v '^\.$$') - -# This variable contains all the actual .xml files covered by the .sources -# file. It obviously is a prerequisite because a page has to be rebuilt if any -# of the .xml files included into it has changed. -SOURCES_REQS = $(shell ./build/source_globber.sh sourceglobs $@ | sed 's;$$;.??.xml;g' ) - -# We simply touch the .sources file. The corresponding .xhtml files in all -# languages depend on the .sources file, so all languages will be rebuilt in -# the main build run. -all: $(SOURCES_FILES) -%.sources: $$(SOURCES_DIRS) $$(SOURCES_REQS) | tagmaps - touch $@ +.PHONY: xmllists +all: xmllists +xmllists: $(SUBDIRS) + @build/make_xmllists.sh diff --git a/build/arguments.sh b/build/arguments.sh index 688f40f18a..2952f90d2e 100755 --- a/build/arguments.sh +++ b/build/arguments.sh @@ -52,20 +52,6 @@ if [ -z "$inc_arguments" ]; then [ "$#" -gt 0 ] && shift 1 && workfile="$1" [ "$#" -gt 0 ] && shift 1 && processor="$1" ;; - sourceglobs) - command="$1$command" - [ "$#" -gt 0 ] && shift 1 && sourcesfile="$1" - ;; - lang_sources) - command="$1$command" - [ "$#" -gt 0 ] && shift 1 && sourceglobfile="$1" - [ "$#" -gt 0 ] && shift 1 && lang="$1" - ;; - cast_refglobs) - command="$1$command" - [ "$#" -gt 0 ] && shift 1 && globfile="$1" - [ "$#" -gt 0 ] && shift 1 && reffile="$1" - ;; wakeup_news) command="$1$command" [ "$#" -gt 0 ] && shift 1 && today="$1" @@ -99,9 +85,6 @@ if [ -z "$inc_arguments" ]; then process_file) [ -z "$workfile" ] && die "Need at least input file" ;; build_xmlstream) [ -z "$workfile" ] && die "Missing xhtml file name" ;; tree_maker) [ -z "$target" ] && die "Missing target location" ;; - sourceglobs) [ -z "$sourcesfile" ] && die "Missing .sources file" ;; - lang_sources) [ -z "$sourceglobfile" -o -z "$lang" ] && die "Need source globfile and language" ;; - cast_refglobs) [ -z "$globfile" -o -z "$reffile" ] && die "Need globfile and reffile" ;; wakeup_news) true;; *help*) print_help; exit 0 ;; *) die "Urecognised command or no command given" ;; diff --git a/build/build_main.sh b/build/build_main.sh index eff32db780..423057c3a0 100755 --- a/build/build_main.sh +++ b/build/build_main.sh @@ -69,8 +69,5 @@ case "$command" in process_file) process_file "$workfile" "$processor" ;; build_xmlstream) build_xmlstream "$(get_shortname "$workfile")" "$(get_language "$workfile")" ;; tree_maker) tree_maker "$tree" "$target" ;; - sourceglobs) sourceglobs "$sourcesfile" ;; - lang_sources) lang_sources "$sourceglobfile" "$lang" ;; - cast_refglobs) cast_refglobs "$globfile" "$reffile" ;; wakeup_news) wakeup_news "$today" ;; esac diff --git a/build/buildrun.sh b/build/buildrun.sh index 0d45d05f94..2d42d513bc 100755 --- a/build/buildrun.sh +++ b/build/buildrun.sh @@ -11,8 +11,8 @@ build_into(){ [ -n "$statusdir" ] && cp "$basedir/build/status.html.sh" "$statusdir/index.cgi" printf %s "$start_time" |logstatus start_time - [ -s "$(logname lasterror)" ] && rm "$(logname lasterror)" - [ -s "$(logname debug)" ] && truncate -s 0 "$(logname debug)" + [ -f "$(logname lasterror)" ] && rm "$(logname lasterror)" + [ -f "$(logname debug)" ] && rm "$(logname debug)" forcelog Makefile diff --git a/build/make_tagmaps.sh b/build/make_tagmaps.sh deleted file mode 100755 index 443734d1d9..0000000000 --- a/build/make_tagmaps.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/sh -# ----------------------------------------------------------------------------- -# Update tagmaps (tools/tagmaps/*.map) and tag list pages (tags/tagged-*) -# ----------------------------------------------------------------------------- -# This script collects all content from all XML files in the source -# directory, and from that creates or updates the following files: -# -# tools/tagmaps/.map - a list of all XML files containing that tag. It is -# used by the "premake" Makefile via "build/source_globber.sh sourceglobs" to -# determine the XML files covered by each .sources file. -# -# tags/tagged-.en.xhtml - a source file which will be built by the -# standard build process into a web page listing all news and events with -# this tag. -# -# tags/tagged-.sources - the pattern list of XML files to be included when -# building that web page. -# -# Each of these files is only touched when the actual file list for a tag -# changes, so the makefile can determine which taglist web pages must be -# rebuilt. -# -# Changing or removing tags in XML files is also considered, in which case a -# file is removed from the map, and the taglist web page source files are -# touched so the build script will rebuild the corresponding web page. -# -# When a tag has been removed from the last XML file where it has been used, -# all the files listed above are correctly deleted. -# ----------------------------------------------------------------------------- - -set -e - -echo "Updating tag maps" - -# ----------------------------------------------------------------------------- -# Make sure temporary directory is empty -# ----------------------------------------------------------------------------- - -rm -rf /tmp/tagmaps -mkdir /tmp/tagmaps - -# ----------------------------------------------------------------------------- -# Create a complete and current map of which tag is used in which files -# ----------------------------------------------------------------------------- - -echo "* Collecting list of files for each tag" - -for xml_file in `find * -name '*.xml' | xargs grep -l '' | sort`; do - xsltproc build/xslt/get_tags.xsl "${xml_file}" | while read raw_tag; do - tag=`echo "${raw_tag}" | tr -d ' +-/:_' | tr '[:upper:]' '[:lower:]'` - echo ${xml_file} >> "/tmp/tagmaps/${tag}.map" - done -done - -# ----------------------------------------------------------------------------- -# Update only those map files where a change has happened (an XML file been -# added or removed) so make can later see what has changed since the last build -# ----------------------------------------------------------------------------- - -echo "* Checking for updated tags" - -for map_file in `ls /tmp/tagmaps`; do - if ! cmp --quiet "/tmp/tagmaps/${map_file}" "tools/tagmaps/${map_file}"; then - tag=`basename "${map_file}" .map` - echo "* Tag ${tag} has been updated." - cp "/tmp/tagmaps/${map_file}" "tools/tagmaps/${map_file}" - cp "tags/tagged.en.xhtml" "tags/tagged-${tag}.en.xhtml" - echo "news/*/news:[${tag}]" >> "tags/tagged-${tag}.sources" - echo "news/*/.news:[${tag}]" >> "tags/tagged-${tag}.sources" - echo "news/nl/nl:[${tag}]" >> "tags/tagged-${tag}.sources" - echo "news/nl/.nl:[${tag}]" >> "tags/tagged-${tag}.sources" - echo "events/*/events:[${tag}]" > "tags/tagged-${tag}.sources" - echo "d_day:[]" >> "tags/tagged-${tag}.sources" - fi -done - -# ----------------------------------------------------------------------------- -# Remove the map files for tags which have been completely deleted -# ----------------------------------------------------------------------------- - -echo "* Checking for deleted tags" - -for map_file in `ls tools/tagmaps | grep '\.map'`; do - if [ ! -f "/tmp/tagmaps/${map_file}" ]; then - tag=`basename "${map_file}" .map` - echo "* Tag ${tag} has been deleted." - rm "tools/tagmaps/${map_file}" - rm "tags/tagged-${tag}.en.xhtml" - rm "tags/tagged-${tag}.sources" - fi -done - -# ----------------------------------------------------------------------------- -# Remove the temporary directory -# ----------------------------------------------------------------------------- - -echo "* Cleaning up" - -rm -rf /tmp/tagmaps diff --git a/build/make_xmllists.sh b/build/make_xmllists.sh new file mode 100755 index 0000000000..db366abac9 --- /dev/null +++ b/build/make_xmllists.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# ----------------------------------------------------------------------------- +# Update XML filelists (*.xmllist) and tag list pages (tags/tagged-*) +# ----------------------------------------------------------------------------- +# From all content from all XML files, and from all .sources filelists, +# this script creates or updates the following files: +# +# */..xmllist - a list of all XML files matching the patterns in the +# corresponding */.sources list. +# +# tags/tagged-.en.xhtml - a source file which will be built by the +# standard build process into a web page listing all news and events with +# this tag. +# +# Each of these files is only touched when the actual content has changed, +# so the build makefile can determine which web pages must be rebuilt. +# +# Changing or removing tags in XML files is also considered, in which case a +# file is removed from the .xmllist files. +# +# When a tag has been removed from the last XML file where it has been used, +# the tagged-* are correctly deleted. +# ----------------------------------------------------------------------------- + +set -e +set -o pipefail + +# ----------------------------------------------------------------------------- +# Make sure temporary directory is empty +# ----------------------------------------------------------------------------- + +rm -rf /tmp/tagmaps +mkdir /tmp/tagmaps + +# ----------------------------------------------------------------------------- +# Create a complete and current map of which tag is used in which files +# ----------------------------------------------------------------------------- + +echo "* Generating tag maps" + +for xml_file in `find * -name '*.xml' | xargs grep -l '' | sort`; do + xsltproc "build/xslt/get_tags.xsl" "${xml_file}" | while read raw_tag; do + tag=`echo "${raw_tag}" | tr -d ' +-/:_' | tr '[:upper:]' '[:lower:]'` + echo ${xml_file} | sed -r 's;\...\.xml$;;' >> "/tmp/tagmaps/${tag}" + done +done + +for tag in `ls /tmp/tagmaps`; do + echo "d_day" >> "/tmp/tagmaps/${tag}" + sort -u "/tmp/tagmaps/${tag}" > "/tmp/tagmaps/tmp" + mv "/tmp/tagmaps/tmp" "/tmp/tagmaps/${tag}" +done + +# ----------------------------------------------------------------------------- +# Update only those files where a change has happened (an XML file been added +# or removed) so make can later see what has changed since the last build +# ----------------------------------------------------------------------------- + +for tag in `ls /tmp/tagmaps`; do + if ! cmp --quiet "/tmp/tagmaps/${tag}" "tags/.tagged-${tag}.xmllist"; then + echo "* Updating tag ${tag}" + cp "tags/tagged.en.xhtml" "tags/tagged-${tag}.en.xhtml" + cp "/tmp/tagmaps/${tag}" "tags/.tagged-${tag}.xmllist" + fi +done + +# ----------------------------------------------------------------------------- +# Remove the map files for tags which have been completely deleted +# ----------------------------------------------------------------------------- + +for tag in `ls tags | sed -rn 's/tagged-(.*)\.en.xhtml/\1/p'`; do + if [ ! -f "/tmp/tagmaps/${tag}" ]; then + echo "* Deleting tag ${tag}" + rm "tags/tagged-${tag}.en.xhtml" + rm "tags/.tagged-${tag}.xmllist" + fi +done + +# ----------------------------------------------------------------------------- +# Remove the temporary directory +# ----------------------------------------------------------------------------- + +rm -rf /tmp/tagmaps + +# ----------------------------------------------------------------------------- +# Update map files for .sources +# ----------------------------------------------------------------------------- + +all_xml="`find * -name '*.xml' | sed -r 's;\...\.xml$;;' | sort -u`" + +for source_file in `find * -name '*.sources' | sort`; do + cat ${source_file} | while read line; do + pattern=`echo "${line}" | sed -rn 's;(.*):\[.*\]$;\1;p'` + pattern=`echo "${pattern}" | sed -r -e 's;\.;\\.;g' -e 's;\*;.*;g'` + tag=`echo "${line}" | sed -rn 's;.*:\[(.*)\]$;\1;p'` + tag=`echo "${tag}" | tr -d ' +-/:_' | tr '[:upper:]' '[:lower:]'` + + if [ -z "${pattern}" ]; then + continue + fi + + # We append || true so the script doesn't fail if grep finds nothing at all + if [ -n "${tag}" ]; then + cat "tags/.tagged-${tag}.xmllist" + else + echo "${all_xml}" + fi | grep "${pattern}" || true + done | sort -u > "/tmp/xmllist" + + list_file="`dirname ${source_file}`/.`basename ${source_file} .sources`.xmllist" + + if ! cmp --quiet "/tmp/xmllist" "${list_file}"; then + echo "* Updating ${list_file}" + cp "/tmp/xmllist" "${list_file}" + fi + + rm -f /tmp/xmllist +done diff --git a/build/makerules.sh b/build/makerules.sh index 877a99b9ba..6012be13d6 100755 --- a/build/makerules.sh +++ b/build/makerules.sh @@ -45,8 +45,9 @@ xhtml_maker(){ filedir="${shortname%/*}" shortbase="${shortname##*/}" processor="$(get_processor "$shortname")" + list_file="`dirname ${shortname}`/.`basename ${shortname}`.xmllist" - [ -f "${shortname}.sources" ] && sourcesfile="${shortname}.sources" || unset sourcesfile + [ -f "${list_file}" ] && xmllist="${list_file}" || unset xmllist # For speed considerations: avoid all disk I/O in this loop for lang in $(get_languages); do @@ -62,7 +63,7 @@ xhtml_maker(){ cat <"$outfile" || { rm $outfile; exit 1; } MakeEND done @@ -249,7 +250,7 @@ EOF for lang in ${languages}; do cat< \$@ EOF @@ -278,7 +279,7 @@ EOF for lang in ${languages}; do cat< \$@ EOF @@ -291,13 +292,13 @@ EOF # ----------------------------------------------------------------------------- # All files which should just be copied over -COPY_SRC_FILES := \$(shell find \$(INPUTDIR) -type f -not -path '\$(INPUTDIR)/.git/*' -not -name 'Makefile' -not -name '*.sources' -not -name '*.xhtml' -not -name '*.xml' -not -name '*.xsl') +COPY_SRC_FILES := \$(shell find \$(INPUTDIR) -type f -not -path '\$(INPUTDIR)/.git/*' -not -name 'Makefile' -not -name '*.sources' -not -name "*.xmllist" -not -name '*.xhtml' -not -name '*.xml' -not -name '*.xsl') # The same as above, but moved to the output directory COPY_DST_FILES := \$(patsubst \$(INPUTDIR)/%,\$(OUTPUTDIR)/%,\$(COPY_SRC_FILES)) all: \$(COPY_DST_FILES) -\$(COPY_DST_FILEST): \$(OUTPUTDIR)/%: \$(INPUTDIR)/% +\$(COPY_DST_FILES): \$(OUTPUTDIR)/%: \$(INPUTDIR)/% @echo "* Copying file \$*" @cp \$< \$@ @@ -310,7 +311,7 @@ SOURCE_DST_FILES := \$(patsubst \$(INPUTDIR)/%,\$(OUTPUTDIR)/source/%,\$(HTML_SR all: \$(SOURCE_DST_FILES) \$(SOURCE_DST_FILES): \$(OUTPUTDIR)/source/%: \$(INPUTDIR)/% @echo "* Copying source \$*" - cp \$< \$@ + @cp \$< \$@ # ----------------------------------------------------------------------------- # Clean up excess files in target directory diff --git a/build/scaffold.sh b/build/scaffold.sh index b3efbdb252..4f4deb049c 100755 --- a/build/scaffold.sh +++ b/build/scaffold.sh @@ -59,7 +59,7 @@ build_xmlstream(){ \$Author: automatic \$ - $(auto_sources "${shortname}.sources" "$lang") + $(auto_sources "${shortname}" "$lang") $(include_xml "$infile") diff --git a/build/source_globber.sh b/build/source_globber.sh deleted file mode 100755 index 2c23675c2d..0000000000 --- a/build/source_globber.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -basedir="${0%/*}/.." -[ -z "$inc_sources" ] && . "$basedir/build/sources.sh" - -. "$basedir/build/arguments.sh" - -case "$command" in - sourceglobs) sourceglobs "$sourcesfile" ;; - lang_sources) lang_sources "$sourceglobfile" "$lang" ;; - cast_refglobs) cast_refglobs "$globfile" "$reffile" ;; - *) die "Unrecognised command or no command given" ;; -esac diff --git a/build/sources.sh b/build/sources.sh index e938e07ca3..371efa9d89 100755 --- a/build/sources.sh +++ b/build/sources.sh @@ -4,56 +4,25 @@ inc_sources=true [ -z "$inc_misc" ] && . "$basedir/build/misc.sh" [ -z "$inc_xmlfiles" ] && . "$basedir/build/xmlfiles.sh" -sourceglobs(){ - # read a .sources file and glob up referenced xml files for processing in list_sources - sourcesfile="$1" - - [ -f "$sourcesfile" ] && grep ':\[.*\]$' "$sourcesfile" \ - | while read line; do - glob="${line%:\[*\]}" - tags="$(printf %s "$line" \ - | sed -r 's;^.+:\[(.*)\]$;\1;; - y;ABCDEFGHIJKLMNOPQRSTUVWXYZ;abcdefghijklmnopqrstuvwxyz; - s;[-_+ /];;g - s;,; ;g - ')" - - sourcefiles="$(printf '%s\n' "$basedir/"${glob}*.[a-z][a-z].xml)" - - for tag in $tags; do - sourcefiles="$(printf '%s\n' "$sourcefiles" |grep -Ff "$basedir/tools/tagmaps/${tag}.map")" - done - printf '%s\n' "$sourcefiles" - done \ - | sed -rn 's;^(.+).[a-z]{2}.xml$;\1;p' \ - | sort -u -} - list_sources(){ - # read a .sources file and generate a list + # read a .xmllist file and generate a list # of all referenced xml files with preference # for a given language - sourcesfile="$1" + shortname="$1" lang="$2" - # Optional 3rd parameter: preglobbed list of source files - # can lead to speed gain in some cases - [ "$#" -ge 3 ] && \ - sourceglobs="$3" \ - || sourceglobs="$(sourceglobs "$sourcesfile")" - - for base in $sourceglobs; do - echo "${base}".[a-z][a-z].xml "${base}".en.[x]ml "${base}.${lang}".[x]ml - done |sed -rn 's;^(.* )?([^ ]+\.[a-z]{2}\.xml).*$;\2;p' + cat "`dirname ${shortname}`/.`basename ${shortname}`.xmllist" | while read base; do + echo "${basedir}/${base}".[a-z][a-z].xml "${basedir}/${base}".en.[x]ml "${basedir}/${base}.${lang}".[x]ml + done | sed -rn 's;^(.* )?([^ ]+\.[a-z]{2}\.xml).*$;\2;p' } auto_sources(){ # import elements from source files, add file name # attribute to first element included from each file - sourcesfile="$1" + shortname="$1" lang="$2" - list_sources "$sourcesfile" "$lang" \ + list_sources "$shortname" "$lang" \ | while read source; do printf '\n### filename="%s" ###\n%s' "$source" "$(include_xml "$source")" done \ @@ -61,18 +30,3 @@ auto_sources(){ s;\n### (filename="[^\n"]+") ###\n[^<]*(]+>[^<]*)*(<([^/>]+/)*([^/>]+))(/?>);\2\3 \1\6;g; ' } - -lang_sources(){ - sourcesfile="$1" - lang="$2" - - list_sources "$sourcesfile" "$lang" -} - -cast_refglobs(){ - true -} - -cast_langglob(){ - true -} diff --git a/build/xslt/get_tags.xsl b/build/xslt/get_tags.xsl index 44bcd607fe..31161709a0 100644 --- a/build/xslt/get_tags.xsl +++ b/build/xslt/get_tags.xsl @@ -5,7 +5,7 @@ - + diff --git a/tools/tagmaps/README b/tools/tagmaps/README deleted file mode 100644 index d19c7ea1cc..0000000000 --- a/tools/tagmaps/README +++ /dev/null @@ -1,3 +0,0 @@ -This directory is used for auto generated tag maps. -A tag map has the file name of a tag, used anywhere on the site -and contains a list of all files that use this tag.