From d1c05f9f3412d07255a3571fd763d51569cc263f Mon Sep 17 00:00:00 2001 From: paul Date: Wed, 7 Sep 2016 16:35:07 +0000 Subject: [PATCH] run quicker tagging engine in prebuild svn path=/trunk/; revision=34240 --- Makefile | 7 +++++ build/buildrun.sh | 25 ++++++++++------- build/makerules.sh | 61 ++++++----------------------------------- build/source_globber.sh | 2 +- build/sources.sh | 31 ++++++--------------- build/stirrups.sh | 5 ++-- tags/tagged.sources | 8 +++--- tags/tags.sources | 8 +++--- 8 files changed, 51 insertions(+), 96 deletions(-) diff --git a/Makefile b/Makefile index 18a76bc4d5..e40ce8cbab 100644 --- a/Makefile +++ b/Makefile @@ -52,3 +52,10 @@ d_month.en.xml: d_day.en.xml grep -q '$(MONTH)' $@ || echo '$(MONTH)' >$@ d_year.en.xml: d_month.en.xml grep -q '$(YEAR)' $@ || echo '$(YEAR)' >$@ + +.PHONY: SOURCEUPDATES +SOURCEUPDATES: $(shell find ./ -name '*.sources') +SOURCEREQS = $(shell ./build/source_globber.sh sourceglobs $@ |sed -r 's;$$;.??.xml;g') +.SECONDEXPANSION: +%.sources: $$(SOURCEREQS) + touch $@ diff --git a/build/buildrun.sh b/build/buildrun.sh index a1ae84edb0..d3f801b8ac 100755 --- a/build/buildrun.sh +++ b/build/buildrun.sh @@ -63,20 +63,25 @@ svn_build_into(){ exit 0 else logstatus SVNlatest <"$SVNchanges" - regen_globs=false regen_xhtml=false regen_xsldeps=false regen_copy=false - egrep -q '^[^UGR]... .*\.xml' "$SVNchanges" && regen_globs=true - egrep -q '^[^A]... .*\.sources' "$SVNchanges" && regen_globs=true - egrep -q '^A... .*\.xhtml' "$SVNchanges" && regen_globs=true - egrep -q '^[AD]... .*\.sources' "$SVNchanges" && regen_xhtml=true - egrep -q '^[^UGR]... .*\.xhtml' "$SVNchanges" && regen_xhtml=true - egrep -q '^A... .*\.xsl' "$SVNchanges" && regen_xhtml=true - egrep -q '^[^A]... .*\.xsl' "$SVNchanges" && regen_xsldeps=true - sed -r '/.*\.(xml|xsl|xhtml|sources)$/d;/Makefile$/d' "$SVNchanges" \ - | egrep -q '^[^AUGR]... .*' && regen_copy=true + # What to do, if a certain file type gets added, deleted, modified + egrep -q '^[A]... .*\.xml' "$SVNchanges" && true + egrep -q '^[D]... .*\.xml' "$SVNchanges" && true + egrep -q '^[UGR]... .*\.xml' "$SVNchanges" && true + egrep -q '^[A]... .*\.xhtml' "$SVNchanges" && true + egrep -q '^[D]... .*\.xhtml' "$SVNchanges" && regen_xhtml=true + egrep -q '^[UGR]... .*\.xhtml' "$SVNchanges" && true + egrep -q '^[A]... .*\.sources' "$SVNchanges" && regen_xhtml=true + egrep -q '^[D]... .*\.sources' "$SVNchanges" && regen_xhtml=true + egrep -q '^[UGR]... .*\.sources' "$SVNchanges" && true + egrep -q '^[A]... .*\.xsl' "$SVNchanges" && regen_xhtml=true + egrep -q '^[D]... .*\.xsl' "$SVNchanges" && regen_xhtml=true && regen_xsldeps=true + egrep -q '^[UGR]... .*\.xsl' "$SVNchanges" && regen_xsldeps=true + egrep -v '.*(\.xml|\.xsl|\.xhtml|\.sources|Makefile)$' "$SVNchanges" \ + | egrep -q '^[D]... .*' && regen_copy=true build_into $(sed -rn '/.*(Makefile|\.xml)$/d;s;^A... (.+)$;\1;p' "$SVNchanges") fi diff --git a/build/makerules.sh b/build/makerules.sh index 9c246e7687..c692d95184 100755 --- a/build/makerules.sh +++ b/build/makerules.sh @@ -32,45 +32,6 @@ mes(){ | tr '\n' ' ' } -glob_maker(){ - # issue make rules for preglobbed sources files - sourcesfile="$1" - - filedir="\${INPUTDIR}/${sourcesfile}" - filedir="${filedir%/*}" - shortbase="${sourcesfile##*/}" - shortbase="${shortbase%.sources}" - - for lang in $(get_languages); do - globfile="${filedir}/._._${shortbase}.${lang}.sourceglobs" - refglobs="${filedir}/._._${shortbase}.${lang}.refglobs" - cat <"$globfile" -$(mes "$refglobs"): $(mes "$globfile") - \${PGLOBBER} \${PROCFLAGS} cast_refglobs "$globfile" "$refglobs" -MakeEND - done -} - -glob_makers(){ - # generate make rules for globbing all .sources files - # within input tree - sourcefind -name '*.sources' \ - | while read filepath; do - glob_maker "$filepath" - done -} - -glob_additions(){ - printf "%s\n" "$@" \ - | egrep '.+\.sources$' \ - | xargs realpath \ - | while read addition; do - glob_maker "${addition#$input/}" - done -} - xhtml_maker(){ # generate make rules for building html files out of xhtml # account for included xml files and xsl rules @@ -85,9 +46,6 @@ xhtml_maker(){ shortbase="${shortname##*/}" processor="$(get_processor "$shortname")" - langglob="$filedir/._._${shortbase}.langglob" - cast_langglob "$shortname" "$langglob" - [ -f "$shortname".rss.xsl ] && bool_rss=true || bool_rss=false [ -f "$shortname".ics.xsl ] && bool_ics=true || bool_ics=false @@ -100,7 +58,7 @@ xhtml_maker(){ bool_indexname=false fi - [ -f "${shortname}.sources" ] && bool_sourceinc=true || bool_sourceinc=false + [ -f "${shortname}.sources" ] && sourcesfile="${shortname}.sources" || unset sourcesfile # For speed considerations: avoid all disk I/O in this loop for lang in $(get_languages); do @@ -116,23 +74,23 @@ xhtml_maker(){ textsfile="$(get_textsfile "$lang")" fundraisingfile="$(get_fundraisingfile "$lang")" - $bool_sourceinc && sourceglobs="${filedir#./}/._._${shortbase}.${lang}.refglobs" || unset sourceglobs + # $bool_sourceinc && sourceglobs="${filedir#./}/._._${shortbase}.${lang}.refglobs" || unset sourceglobs cat <"$outfile" $(mes "$outlink"): ln -sf "${outbase}" "${outlink}" MakeEND $bool_rss && cat<"$rssfile" MakeEND $bool_ics && cat<"$icsfile" MakeEND $bool_indexname && cat <\${INPUTDIR}/tagmap +# \${INPUTDIR}/tagmap: \$(shell find "$basedir" -name '*.[a-z][a-z].xml') +# find "$basedir" -name '*.[a-z][a-z].xml' |xargs \${PGLOBBER} \${PROCFLAGS} map_tags >\${INPUTDIR}/tagmap MakeHead forcelog Make_globs; Make_globs="$(logname Make_globs)" @@ -287,9 +245,6 @@ MakeHead trap "trap - 0 2 3 6 9 15; killall \"${0##*/}\"" 0 2 3 6 9 15 - [ "$regen_globs" = false -a -s "$Make_globs" ] && \ - glob_additions "$@" >>"$Make_globs" \ - || glob_makers >"$Make_globs" & [ "$regen_xslt" = false -a -s "$Make_xslt" ] && \ xslt_additions "$@" >>"$Make_xslt" \ || xslt_makers >"$Make_xslt" & @@ -310,6 +265,6 @@ MakeHead wait trap - 0 2 3 6 9 15 - cat "$Make_globs" "$Make_xslt" "$Make_copy" "$Make_sourcecopy" + cat "$Make_xslt" "$Make_copy" "$Make_sourcecopy" } diff --git a/build/source_globber.sh b/build/source_globber.sh index fe5d566224..c82ec1de71 100755 --- a/build/source_globber.sh +++ b/build/source_globber.sh @@ -10,5 +10,5 @@ case "$command" in sourceglobs) sourceglobs "$sourcesfile" ;; lang_sources) lang_sources "$sourceglobfile" "$lang" ;; cast_refglobs) cast_refglobs "$globfile" "$reffile" ;; - *) die "Urecognised command or no command given" ;; + *) die "Unrecognised command or no command given" ;; esac diff --git a/build/sources.sh b/build/sources.sh index 9c9a780ee1..501e62f92b 100755 --- a/build/sources.sh +++ b/build/sources.sh @@ -11,7 +11,8 @@ validate_tagmap(){ } map_tags(){ - for xml in "$@"; do + grep -l '' "$@" \ + | while read xml; do printf '%s ' "$xml" unicat "$xml" \ | sed -rn ':a;N;$!ba @@ -34,29 +35,15 @@ tagging_sourceglobs(){ | while read line; do glob="${line%:\[*\]}" tags="$(printf %s "$line" |sed -r 's;^.+:\[(.*)\]$;\1;;s; ;+;g;s;,; ;g')" - - # Input file must match *all* tags from line definition. - # Build a sed expression, that performs conjunctive match - # at once, e.g. to match all of the tags 'spam', 'eggs', - # and 'bacon' the expression will have roughly the form - # "/spam/{/eggs/{/bacon/{p}}}" - - match="$(printf '%s' "$glob" |sed -r 's;\*;.*;g;s;\?;.;g')" - matchline="s;^(${basedir}/${match}.*)\.[a-z]{2}\.xml .*$;\1;p" - for tag in $tags ; do - matchline="/ $tag( |$)/{${matchline}}" - done - if [ -z "$tags" ]; then - # save the i/o if tags are empty, i.e. always match - printf '%s \n' "$basedir/"${glob}*.[a-z][a-z].xml |sed -rn "$matchline" - elif [ -f "$basedir/tagmap" ]; then - sed -rn "$matchline" <"$basedir/tagmap" - else - map_tags "$basedir/"${glob}*.[a-z][a-z].xml \ - | sed -rn "$matchline" - fi + sourcefiles="$(printf '%s\n' "$basedir/"${glob}*.[a-z][a-z].xml)" + + for tag in $tags; do + sourcefiles="$(printf '%s\n' "$sourcefiles" |grep -Ff "$basedir/tools/tagmaps/${tag}.map")" + done + printf '%s\n' "$sourcefiles" done \ + | sed -rn 's;^(.+).[a-z]{2}.xml$;\1;p' \ | sort -u } diff --git a/build/stirrups.sh b/build/stirrups.sh index 0606f1eb3e..2f17b9102c 100755 --- a/build/stirrups.sh +++ b/build/stirrups.sh @@ -2,13 +2,14 @@ inc_stirrups=true [ -z "$inc_misc" ] && . "$basedir/build/misc.sh" -[ -z "$inc_sources" ] && . "$basedir/build/sources.sh" +# [ -z "$inc_sources" ] && . "$basedir/build/sources.sh" validate_caches(){ # outdate / remove cache files if necessary # hook functions here as required - validate_tagmap # hook from sources.sh + # validate_tagmap # hook from sources.sh + true } dir_maker(){ diff --git a/tags/tagged.sources b/tags/tagged.sources index 96b0b89bd2..7c87392044 100755 --- a/tags/tagged.sources +++ b/tags/tagged.sources @@ -1,4 +1,4 @@ -news/*/news:[.+] -news/generated_xml/:[.+] -news/nl/nl:[.+] -events/*/event:[.+] +news/*/news:[] +news/generated_xml/:[] +news/nl/nl:[] +events/*/event:[] diff --git a/tags/tags.sources b/tags/tags.sources index 96b0b89bd2..7c87392044 100755 --- a/tags/tags.sources +++ b/tags/tags.sources @@ -1,4 +1,4 @@ -news/*/news:[.+] -news/generated_xml/:[.+] -news/nl/nl:[.+] -events/*/event:[.+] +news/*/news:[] +news/generated_xml/:[] +news/nl/nl:[] +events/*/event:[]