FSFE/fsfe-website
25
31
Fork 91
Code Issues 64 Pull Requests 5 Wiki Activity

parallel rule generation

Browse Source 
svn path=/trunk/; revision=31276
This commit is contained in:
paul 2015-05-28 14:55:27 +00:00
parent 08e51a9000
commit 78efe8bd06
5 changed files with 106 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
build/buildrun.sh
View File

@ -2,18 +2,16 @@
inc_buildrun=true
[ -z "$inc_makerules" ] && . "$basedir/build/makerules.sh"
[ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
[ -z "$inc_stirrups" ] && . "$basedir/build/stirrups.sh"
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
svn_build(){
svn_update(){
run_make=true
regen_globs=false
regen_xsldeps=false
regen_xhtml=false
regen_copy=false
forcelog additions
svn update "$basedir" 2>&1 \
| logstatus SVNchanges \
| while read update; do
@ -42,15 +40,20 @@ svn_build(){
????" "*) regen_copy=true;;
*) true;;
esac
done \
| cut -c6- \
done | cut -c6-
}
svn_build_into(){
svn_update \
| logstatus additions
build_into
}
build_into(){
ncpu="$(cat /proc/cpuinfo |grep ^processor |wc -l)"
ncpu="$(grep -c ^processor /proc/cpuinfo)"
forcelog manifest
forcelog Makefile
make -j $ncpu -C "$basedir" \
| logstatus premake
@ -59,10 +62,11 @@ build_into(){
tree_maker "$basedir" "$target" \
| logstatus Makefile \
| build_manifest "$(logname manifest)" \
| make -j $ncpu -f - \
| logstatus buildlog
remove_orphans "$target" <"$(logname manifest)" \
| build_manifest \
| logstatus manifest \
| remove_orphans "$target" \
| logstatus removed
make -j $ncpu -f "$(logname Makefile)" \
| logstatus buildlog
}

29
build/logging.sh Executable file
View File

@ -0,0 +1,29 @@
#!/bin/sh
inc_logging=true
logname(){
name="$1"
if [ -w "$statusdir" ] && touch "$statusdir/$name"; then
echo "$statusdir/$name"
elif echo "$forcedlog" |egrep -q "^${name}=.+"; then
echo "$forcedlog" \
| sed -rn "s;^${name}=;;p"
else
echo /dev/null
fi
}
forcelog(){
name="$1"
[ "$(logname "$name")" = "/dev/null" ] \
&& forcedlog="$forcedlog\n${name}=$(tempfile -p w3bld -s $$)"
}
trap "trap - 0 2 3 6 9 15; find \"${TMPDIR:-/tmp}/\" -maxdepth 1 -user \"$USER\" -name \"w3bld*$$\" -delete" 0 2 3 6 9 15
logstatus(){
# pipeline atom to write data streams into a log file
tee "$(logname "$1")"
}

97
build/makerules.sh
View File

@ -8,6 +8,10 @@ inc_makerules=true
[ -z "$inc_languages" ] && . "$basedir/build/languages.sh"
[ -z "$inc_sources" ] && . "$basedir/build/sources.sh"
sourcefind() {
find "$input" -name .svn -prune -o -type f "$@" -printf '%P\n'
}
mes(){
# make escape... escape a filename for make syntax
# possibly not complete
@ -39,6 +43,12 @@ $(mes "$globfile"): $(mes "$sourceglobfile")
MakeEND
done
}
glob_makers(){
sourcefind -name '*.sources' \
| while read filepath; do
glob_maker "$input/$filepath"
done
}
xhtml_maker(){
# generate make rules for building html files out of xhtml
@ -108,6 +118,14 @@ $(mes "$outpath/index.html.$lang"):
MakeEND
done
}
xhtml_makers(){
sourcefind -name '*.[a-z][a-z].xhtml' \
| sed -r "s;^(.+)\.[a-z][a-z]\.xhtml$;\1;" \
| sort -u \
| while read shortpath; do
xhtml_maker "$input/$shortpath" "$output/$(dirname "$shortpath")"
done
}
copy_maker(){
# generate make rule for copying a plain file
@ -121,6 +139,13 @@ $(mes "$outfile"): $(mes "$infile")
cp "$infile" "$outfile"
MakeEND
}
copy_makers(){
sourcefind \! -name 'Makefile' \! -name '*.sourceglobs' \! -name '*.sources' \
\! -name '*.xhtml' \! -name '*.xml' \! -name '*.xsl' \
| while read filepath; do
copy_maker "$input/$filepath" "$output/$(dirname "$filepath")"
done
}
xslt_dependencies(){
file="$1"
@ -144,16 +169,17 @@ $(mes "$file"): $deps
touch "$file"
MakeEND
}
xslt_makers(){
sourcefind -name '*.xsl' \
| while read filepath; do
xslt_maker "$input/$filepath"
done
}
xhtml_rules(){
input="$(echo "$1" |sed -r 's:/$::')"
output="$(echo "$2" |sed -r 's:/$::')"
find "$input" -type f -and -name '*.[a-z][a-z].xhtml' \
| sed -r "/(^|\/)\.svn($|\/)|^\.\.$/d;s;^$input/*(.+)\.[a-z]{2}\.xhtml$;\1;" \
| sort -u \
| while read shortpath; do
xhtml_maker "${input}/$shortpath" "$output/$(dirname "$shortpath")"
copy_sources(){
sourcefind -name '*.xhtml' \
| while read filepath; do
copy_maker "$input/$filepath" "$output/source/$(dirname "$filepath")"
done
}
@ -173,41 +199,26 @@ tree_maker(){
PROCFLAGS = --source "$basedir" --statusdir "$statusdir" --domain "$domain"
MakeHead
find "$input" -type f -name '*.sources' \
| sed -r "/(^|\/)\.svn($|\/)|^\.\.$/d;s;^$input/*;;" \
| while read filepath; do
glob_maker "$input/$filepath"
done \
| logstatus Make_globs
forcelog Make_globs; Make_globs="$(logname Make_globs)"
forcelog Make_xslt; Make_xslt="$(logname Make_xslt)"
forcelog Make_copy; Make_copy="$(logname Make_copy)"
forcelog Make_sourcecopy; Make_sourcecopy="$(logname Make_sourcecopy)"
forcelog Make_xhtml; Make_xhtml="$(logname Make_xhtml)"
find "$input" -type f -name '*.xsl' \
| sed -r "/(^|\/)\.svn($|\/)|^\.\.$/d;s;^$input/*;;" \
| while read filepath; do
xslt_maker "$input/$filepath"
done \
| logstatus Make_xslt
[ "$regen_globs" != true -a -s "$Make_globs" ] \
|| glob_makers >"$Make_globs" &
[ "$regen_xslt" != true -a -s "$Make_xslt" ] \
|| xslt_makers >"$Make_xslt" &
[ "$regen_copy" != true -a -s "$Make_copy" ] \
|| copy_makers >"$Make_copy" &
[ "$regen_xhtml" != true -a -s "$Make_sourcecopy" ] \
|| copy_sources >"$Make_sourcecopy" &
[ "$regen_xhtml" != true -a -s "$Make_xhtml" ] \
|| xhtml_makers >"$Make_xhtml" &
find "$input" -type f -name '*.xhtml' \
| sed -r "/(^|\/)\.svn($|\/)|^\.\.$/d;s;^$input/*;;" \
| while read filepath; do
copy_maker "$input/$filepath" "$output/source/$(dirname "$filepath")"
done \
| logstatus Make_sourcecopy
find "$input" -type f \
\! -name 'Makefile' \
\! -name '*.sourceglobs' \
\! -name '*.sources' \
\! -name '*.xml' \
\! -name '*.xsl' \
\! -name '*.xhtml' \
| sed -r "/(^|\/)\.svn($|\/)|^\.\.$/d;s;^$input/*;;" \
| while read filepath; do
copy_maker "$input/$filepath" "$output/$(dirname "$filepath")"
done \
| logstatus Make_copy
xhtml_rules "$input" "$output" \
| logstatus Make_xhtml
trap "trap - 0 2 3 6 9 15; killall \"$(basename "$0")\"" 0 2 3 6 9 15
wait
trap - 0 2 3 6 9 15
cat "$Make_globs" "$Make_xslt" "$Make_copy" "$Make_sourcecopy" "$Make_xhtml"
}

43
build/misc.sh
View File

@ -1,6 +1,7 @@
#!/bin/sh
inc_misc=true
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
print_help(){
cat "$basedir/build/HELP"
@ -14,51 +15,13 @@ debug(){
dbg_file=/dev/stderr
if [ "$#" -ge 1 ]; then
echo "$*" >>"$dbg_file"
echo "$@" >>"$dbg_file"
else
tee -a "$dbg_file"
fi
}
logname(){
name="$1"
if [ -w "$statusdir" ] && touch "$statusdir/$name"; then
echo "$statusdir/$name"
elif echo "$forcedlog" |egrep -q "^${name}=.+"; then
echo "$forcedlog" |sed -rn "s;^${name}=;;p"
else
echo /dev/null
fi
}
rmforcedlog(){
echo "$forcedlog" \
| while read logfile; do
[ -f "$logfile" ] && rm "$logfile"
done
}
forcelog(){
name="$1"
[ -n "$(logname "$name")" ] \
|| forcedlog="$forcedlog\n${name}=$(tempfile)"
trap rmforcedlog 0
}
logstatus(){
# pipeline atom to write data streams into a log file
# log file will be created inside statusdir
# if statusdir is not enabled, we won't log to a file
file="$(logname "$1")"
if [ -f "$file" ]; then
tee "$file"
else
cat
fi
}
print_error(){
echo "Error: $*" |logstatus lasterror >/dev/stderr
echo "Error: $@" |logstatus lasterror >/dev/stderr
echo "Run '$0 --help' to see usage instructions" >/dev/stderr
}

7
build/stirrups.sh
View File

@ -23,11 +23,10 @@ dir_maker(){
}
build_manifest(){
# pass Makefile throug on pipe and generate
# read a Makefile from stdin and generate
# list of all make tagets
outfile="$1"
sed -nr 'p;s;/\./;/;g;s;\\ ; ;g;s;([^:]+) :.*;\1;w'"$outfile"
sed -nr 's;/\./;/;g;s;\\ ; ;g;s;([^:]+) :.*;\1;p'
}
remove_orphans(){
@ -44,7 +43,7 @@ remove_orphans(){
# We use 'uniq -u' to drop those from the list.
# Remaining single files exist only in the tree and are to be removed
(find "$tree" -type f -or -type l; cat) \
(find "$tree" -type f -o -type l; cat) \
| sort \
| uniq -u \
| while read file; do