Browse Source

run quicker tagging engine in prebuild

svn path=/trunk/; revision=34240
tags/stw2018
paul 3 years ago
parent
commit
d1c05f9f34
8 changed files with 51 additions and 96 deletions
  1. 7
    0
      Makefile
  2. 15
    10
      build/buildrun.sh
  3. 8
    53
      build/makerules.sh
  4. 1
    1
      build/source_globber.sh
  5. 9
    22
      build/sources.sh
  6. 3
    2
      build/stirrups.sh
  7. 4
    4
      tags/tagged.sources
  8. 4
    4
      tags/tags.sources

+ 7
- 0
Makefile View File

@@ -52,3 +52,10 @@ d_month.en.xml: d_day.en.xml
grep -q '$(MONTH)' $@ || echo '$(MONTH)' >$@
d_year.en.xml: d_month.en.xml
grep -q '$(YEAR)' $@ || echo '$(YEAR)' >$@

.PHONY: SOURCEUPDATES
SOURCEUPDATES: $(shell find ./ -name '*.sources')
SOURCEREQS = $(shell ./build/source_globber.sh sourceglobs $@ |sed -r 's;$$;.??.xml;g')
.SECONDEXPANSION:
%.sources: $$(SOURCEREQS)
touch $@

+ 15
- 10
build/buildrun.sh View File

@@ -63,20 +63,25 @@ svn_build_into(){
exit 0
else
logstatus SVNlatest <"$SVNchanges"
regen_globs=false
regen_xhtml=false
regen_xsldeps=false
regen_copy=false

egrep -q '^[^UGR]... .*\.xml' "$SVNchanges" && regen_globs=true
egrep -q '^[^A]... .*\.sources' "$SVNchanges" && regen_globs=true
egrep -q '^A... .*\.xhtml' "$SVNchanges" && regen_globs=true
egrep -q '^[AD]... .*\.sources' "$SVNchanges" && regen_xhtml=true
egrep -q '^[^UGR]... .*\.xhtml' "$SVNchanges" && regen_xhtml=true
egrep -q '^A... .*\.xsl' "$SVNchanges" && regen_xhtml=true
egrep -q '^[^A]... .*\.xsl' "$SVNchanges" && regen_xsldeps=true
sed -r '/.*\.(xml|xsl|xhtml|sources)$/d;/Makefile$/d' "$SVNchanges" \
| egrep -q '^[^AUGR]... .*' && regen_copy=true
# What to do, if a certain file type gets added, deleted, modified
egrep -q '^[A]... .*\.xml' "$SVNchanges" && true
egrep -q '^[D]... .*\.xml' "$SVNchanges" && true
egrep -q '^[UGR]... .*\.xml' "$SVNchanges" && true
egrep -q '^[A]... .*\.xhtml' "$SVNchanges" && true
egrep -q '^[D]... .*\.xhtml' "$SVNchanges" && regen_xhtml=true
egrep -q '^[UGR]... .*\.xhtml' "$SVNchanges" && true
egrep -q '^[A]... .*\.sources' "$SVNchanges" && regen_xhtml=true
egrep -q '^[D]... .*\.sources' "$SVNchanges" && regen_xhtml=true
egrep -q '^[UGR]... .*\.sources' "$SVNchanges" && true
egrep -q '^[A]... .*\.xsl' "$SVNchanges" && regen_xhtml=true
egrep -q '^[D]... .*\.xsl' "$SVNchanges" && regen_xhtml=true && regen_xsldeps=true
egrep -q '^[UGR]... .*\.xsl' "$SVNchanges" && regen_xsldeps=true
egrep -v '.*(\.xml|\.xsl|\.xhtml|\.sources|Makefile)$' "$SVNchanges" \
| egrep -q '^[D]... .*' && regen_copy=true

build_into $(sed -rn '/.*(Makefile|\.xml)$/d;s;^A... (.+)$;\1;p' "$SVNchanges")
fi

+ 8
- 53
build/makerules.sh View File

@@ -32,45 +32,6 @@ mes(){
| tr '\n' ' '
}

glob_maker(){
# issue make rules for preglobbed sources files
sourcesfile="$1"

filedir="\${INPUTDIR}/${sourcesfile}"
filedir="${filedir%/*}"
shortbase="${sourcesfile##*/}"
shortbase="${shortbase%.sources}"

for lang in $(get_languages); do
globfile="${filedir}/._._${shortbase}.${lang}.sourceglobs"
refglobs="${filedir}/._._${shortbase}.${lang}.refglobs"
cat <<MakeEND
$(mes "$globfile"): $(mes "\${INPUTDIR}/tagmap" "\${INPUTDIR}/$sourcesfile")
\${PGLOBBER} \${PROCFLAGS} lang_sources "\${INPUTDIR}/$sourcesfile" "$lang" >"$globfile"
$(mes "$refglobs"): $(mes "$globfile")
\${PGLOBBER} \${PROCFLAGS} cast_refglobs "$globfile" "$refglobs"
MakeEND
done
}

glob_makers(){
# generate make rules for globbing all .sources files
# within input tree
sourcefind -name '*.sources' \
| while read filepath; do
glob_maker "$filepath"
done
}

glob_additions(){
printf "%s\n" "$@" \
| egrep '.+\.sources$' \
| xargs realpath \
| while read addition; do
glob_maker "${addition#$input/}"
done
}

xhtml_maker(){
# generate make rules for building html files out of xhtml
# account for included xml files and xsl rules
@@ -85,9 +46,6 @@ xhtml_maker(){
shortbase="${shortname##*/}"
processor="$(get_processor "$shortname")"

langglob="$filedir/._._${shortbase}.langglob"
cast_langglob "$shortname" "$langglob"

[ -f "$shortname".rss.xsl ] && bool_rss=true || bool_rss=false
[ -f "$shortname".ics.xsl ] && bool_ics=true || bool_ics=false

@@ -100,7 +58,7 @@ xhtml_maker(){
bool_indexname=false
fi

[ -f "${shortname}.sources" ] && bool_sourceinc=true || bool_sourceinc=false
[ -f "${shortname}.sources" ] && sourcesfile="${shortname}.sources" || unset sourcesfile

# For speed considerations: avoid all disk I/O in this loop
for lang in $(get_languages); do
@@ -116,23 +74,23 @@ xhtml_maker(){

textsfile="$(get_textsfile "$lang")"
fundraisingfile="$(get_fundraisingfile "$lang")"
$bool_sourceinc && sourceglobs="${filedir#./}/._._${shortbase}.${lang}.refglobs" || unset sourceglobs
# $bool_sourceinc && sourceglobs="${filedir#./}/._._${shortbase}.${lang}.refglobs" || unset sourceglobs

cat <<MakeEND
all: $(mes "$outfile" "$outlink")
$(mes "$outfile"): $(mes "$depfile" "$processor" "$textsen" "$textsfile" "$fundraisingfile" "$menufile" "$sourceglobs" "$langglob")
$(mes "$outfile"): $(mes "$depfile" "$processor" "$textsen" "$textsfile" "$fundraisingfile" "$menufile" "$sourcesfile")
\${PROCESSOR} \${PROCFLAGS} process_file "${infile}" "$(mio "$processor")" "$olang" >"$outfile"
$(mes "$outlink"):
ln -sf "${outbase}" "${outlink}"
MakeEND
$bool_rss && cat<<MakeEND
all: $(mes "$rssfile")
$(mes "$rssfile"): $(mes "$depfile" "${shortname}.rss.xsl" "$textsen" "$textsfile" "$fundraisingfile" "$menufile" "$sourceglobs")
$(mes "$rssfile"): $(mes "$depfile" "${shortname}.rss.xsl" "$textsen" "$textsfile" "$fundraisingfile" "$menufile" "$sourcesfile")
\${PROCESSOR} \${PROCFLAGS} process_file "${infile}" "$(mio "${shortname}.rss.xsl")" "$olang" >"$rssfile"
MakeEND
$bool_ics && cat<<MakeEND
all: $(mes "$icsfile")
$(mes "$icsfile"): $(mes "$depfile" "${shortname}.ics.xsl" "$textsen" "$textsfile" "$fundraisingfile" "$menufile" "$sourceglobs")
$(mes "$icsfile"): $(mes "$depfile" "${shortname}.ics.xsl" "$textsen" "$textsfile" "$fundraisingfile" "$menufile" "$sourcesfile")
\${PROCESSOR} \${PROCFLAGS} process_file "${infile}" "$(mio "${shortname}.ics.xsl")" "$olang" >"$icsfile"
MakeEND
$bool_indexname && cat <<MakeEND
@@ -275,8 +233,8 @@ INPUTDIR = $input
OUTPUTDIR = $output

# cannot store find results in variable because it will result in too many arguments for the shell
\${INPUTDIR}/tagmap: \$(shell find "$basedir" -name '*.[a-z][a-z].xml')
find "$basedir" -name '*.[a-z][a-z].xml' |xargs \${PGLOBBER} \${PROCFLAGS} map_tags >\${INPUTDIR}/tagmap
# \${INPUTDIR}/tagmap: \$(shell find "$basedir" -name '*.[a-z][a-z].xml')
# find "$basedir" -name '*.[a-z][a-z].xml' |xargs \${PGLOBBER} \${PROCFLAGS} map_tags >\${INPUTDIR}/tagmap
MakeHead

forcelog Make_globs; Make_globs="$(logname Make_globs)"
@@ -287,9 +245,6 @@ MakeHead

trap "trap - 0 2 3 6 9 15; killall \"${0##*/}\"" 0 2 3 6 9 15

[ "$regen_globs" = false -a -s "$Make_globs" ] && \
glob_additions "$@" >>"$Make_globs" \
|| glob_makers >"$Make_globs" &
[ "$regen_xslt" = false -a -s "$Make_xslt" ] && \
xslt_additions "$@" >>"$Make_xslt" \
|| xslt_makers >"$Make_xslt" &
@@ -310,6 +265,6 @@ MakeHead
wait
trap - 0 2 3 6 9 15

cat "$Make_globs" "$Make_xslt" "$Make_copy" "$Make_sourcecopy"
cat "$Make_xslt" "$Make_copy" "$Make_sourcecopy"
}


+ 1
- 1
build/source_globber.sh View File

@@ -10,5 +10,5 @@ case "$command" in
sourceglobs) sourceglobs "$sourcesfile" ;;
lang_sources) lang_sources "$sourceglobfile" "$lang" ;;
cast_refglobs) cast_refglobs "$globfile" "$reffile" ;;
*) die "Urecognised command or no command given" ;;
*) die "Unrecognised command or no command given" ;;
esac

+ 9
- 22
build/sources.sh View File

@@ -11,7 +11,8 @@ validate_tagmap(){
}

map_tags(){
for xml in "$@"; do
grep -l '</tag>' "$@" \
| while read xml; do
printf '%s ' "$xml"
unicat "$xml" \
| sed -rn ':a;N;$!ba
@@ -34,29 +35,15 @@ tagging_sourceglobs(){
| while read line; do
glob="${line%:\[*\]}"
tags="$(printf %s "$line" |sed -r 's;^.+:\[(.*)\]$;\1;;s; ;+;g;s;,; ;g')"
# Input file must match *all* tags from line definition.
# Build a sed expression, that performs conjunctive match
# at once, e.g. to match all of the tags 'spam', 'eggs',
# and 'bacon' the expression will have roughly the form
# "/spam/{/eggs/{/bacon/{p}}}"
match="$(printf '%s' "$glob" |sed -r 's;\*;.*;g;s;\?;.;g')"
matchline="s;^(${basedir}/${match}.*)\.[a-z]{2}\.xml .*$;\1;p"
for tag in $tags ; do
matchline="/ $tag( |$)/{${matchline}}"
done

if [ -z "$tags" ]; then
# save the i/o if tags are empty, i.e. always match
printf '%s \n' "$basedir/"${glob}*.[a-z][a-z].xml |sed -rn "$matchline"
elif [ -f "$basedir/tagmap" ]; then
sed -rn "$matchline" <"$basedir/tagmap"
else
map_tags "$basedir/"${glob}*.[a-z][a-z].xml \
| sed -rn "$matchline"
fi
sourcefiles="$(printf '%s\n' "$basedir/"${glob}*.[a-z][a-z].xml)"
for tag in $tags; do
sourcefiles="$(printf '%s\n' "$sourcefiles" |grep -Ff "$basedir/tools/tagmaps/${tag}.map")"
done
printf '%s\n' "$sourcefiles"
done \
| sed -rn 's;^(.+).[a-z]{2}.xml$;\1;p' \
| sort -u
}


+ 3
- 2
build/stirrups.sh View File

@@ -2,13 +2,14 @@

inc_stirrups=true
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
[ -z "$inc_sources" ] && . "$basedir/build/sources.sh"
# [ -z "$inc_sources" ] && . "$basedir/build/sources.sh"

validate_caches(){
# outdate / remove cache files if necessary
# hook functions here as required

validate_tagmap # hook from sources.sh
# validate_tagmap # hook from sources.sh
true
}

dir_maker(){

+ 4
- 4
tags/tagged.sources View File

@@ -1,4 +1,4 @@
news/*/news:[.+]
news/generated_xml/:[.+]
news/nl/nl:[.+]
events/*/event:[.+]
news/*/news:[]
news/generated_xml/:[]
news/nl/nl:[]
events/*/event:[]

+ 4
- 4
tags/tags.sources View File

@@ -1,4 +1,4 @@
news/*/news:[.+]
news/generated_xml/:[.+]
news/nl/nl:[.+]
events/*/event:[.+]
news/*/news:[]
news/generated_xml/:[]
news/nl/nl:[]
events/*/event:[]

Loading…
Cancel
Save