#1218 WIP: benchmark build system and improve performance

Open
jzarl wants to merge 4 commits from jzarl/fsfe-website:l-benchmark into master
  1. +6
    -0
      build/buildrun.sh
  2. +1
    -0
      build/misc.sh
  3. +31
    -0
      tools/inc_profiling.sh
  4. +53
    -15
      tools/update_xmllists.sh

+ 6
- 0
build/buildrun.sh View File

@ -19,10 +19,12 @@ build_into(){
[ -f "$(logname debug)" ] && rm "$(logname debug)"
{
prof_start phase1
echo "Starting phase 1" \
&& make --silent --directory="$basedir" build_env="${build_env:-development}" 2>&1 \
&& echo "Finishing phase 1" \
|| die "Error during phase 1"
prof_elapsed phase1 "Phase 1" >&2
} | t_logstatus phase_1 || exit 1
dir_maker "$basedir" "$stagedir" || exit 1
@ -30,15 +32,19 @@ build_into(){
forcelog Makefile
{
prof_start phase2gen
tree_maker "$basedir" "$stagedir" 2>&1 \
|| die "Error during phase 2 Makefile generation"
prof_elapsed phase2gen "Phase 2 Makefile generation" >&2
} > "$(logname Makefile)" || exit 1
{
prof_start phase2 "Phase 2"
echo "Starting phase 2" \
&& make --silent --jobs=$ncpu --file="$(logname Makefile)" 2>&1 \
&& echo "Finishing phase 2" \
|| die "Error during phase 2"
prof_elapsed phase2 "Phase 2" >&2
} | t_logstatus phase_2 || exit 1
if [ "$stagedir" != "$target" ]; then


+ 1
- 0
build/misc.sh View File

@ -2,6 +2,7 @@
inc_misc=true
[ -z "$inc_logging" ] && . "$basedir/build/logging.sh"
. "$basedir/tools/inc_profiling.sh"
print_help(){
cat "$basedir/build/HELP"


+ 31
- 0
tools/inc_profiling.sh View File

@ -0,0 +1,31 @@
if [ -z "$inc_profiling" ]
then
inc_profiling=true
declare -A TIMER_START
prof_start()
# prof_start ID
# Start timer for ID
{
if [ -n "$PROFILE" ]
then
local id="$1"
TIMER_START[$id]=$( date +%s.%n )
fi
}
prof_elapsed()
# prof_elapsed ID [LABEL]
# Print elapsed time for timer ID.
# Optionally, a user-friendly LABEL can be supplied to be displayed instead of the ID.
{
if [ -n "$PROFILE" ]
then
local id="$1"
local label="${2:-$id}"
local timer_end=$( date +%s.%N )
local elapsed=$(echo "$timer_end - ${TIMER_START[$id]}" | bc -l)
echo "Elapsed time ($label): ${elapsed%%???????} seconds"
fi
}
fi

+ 53
- 15
tools/update_xmllists.sh View File

@ -23,6 +23,8 @@
#
# When a tag has been removed from the last XML file where it has been used,
# the tagged-* are correctly deleted.
#
# Use "export PROFILE=1" to enable profiling output.
# -----------------------------------------------------------------------------
set -e
@ -34,6 +36,22 @@ nextyear=$(date --date="next year" +%Y)
thisyear=$(date --date="this year" +%Y)
lastyear=$(date --date="last year" +%Y)
. "$(dirname "$0")/inc_profiling.sh"
parallelize()
# parallelize command with GNU parallel, if available
# The arguments are run for each parameter provided on stdin, similar to xargs.
{
if [ -z "$FORCE_SERIAL" ] && which parallel >/dev/null
then
parallel --will-cite "$@"
else
# serial execution
while read arg ; do
"$@" "$arg"
done
fi
}
# -----------------------------------------------------------------------------
# Make sure temporary directory is empty
# -----------------------------------------------------------------------------
@ -54,21 +72,33 @@ mkdir "${taglabels}"
echo "* Generating tag maps"
for xml_file in $(find * -name '*.??.xml' -not -path 'tags/*' | xargs grep -l '</tag>' | sort); do
xsltproc "build/xslt/get_tags.xsl" "${xml_file}" | while read tag label; do
# Add file to list of files by tag name
echo "${xml_file%.??.xml}" >> "${tagmaps}/${tag}"
# Store label by language and tag name
xml_base=${xml_file%.xml}
language=${xml_base##*.}
if [ "${language}" -a "${label}" ]; then
mkdir -p "${taglabels}/${language}"
# Always overwrite so the newest news item will win.
echo "${label}" > "${taglabels}/${language}/${tag}"
fi
done
done
prof_start tagmaps
generate_tag_map()
# generate_tag_map LANGUAGE
{
local language="$1"
for xml_file in $(find * -name "*.$language.xml" -not -path 'tags/*' | xargs grep -l '</tag>' | sort); do
xsltproc "build/xslt/get_tags.xsl" "${xml_file}" | while read tag label; do
# Add file to list of files by tag name
echo "${xml_file%.??.xml}" >> "${tagmaps}/${tag}"
# Store label by language and tag name
xml_base=${xml_file%.xml}
if [ "${label}" ]; then
mkdir -p "${taglabels}/${language}"
# Always overwrite so the newest news item will win.
echo "${label}" > "${taglabels}/${language}/${tag}"
fi
done
done
}
export tagmaps taglabels
export -f generate_tag_map
find * -name '*.??.xml' | sed 's/.*\.\(..\)\.xml/\1/' | sort -u | parallelize generate_tag_map
prof_elapsed tagmaps "tag maps" >&2
# -----------------------------------------------------------------------------
# Make sure that each file only appears once per tag map
@ -111,12 +141,14 @@ for tag in $(ls -a "tags" | sed -rn 's/.tagged-(.*)\.xmllist/\1/p'); do
rm "tags/.tagged-${tag}.xmllist"
fi
done
prof_elapsed tagmaps "tag maps total" >&2
# -----------------------------------------------------------------------------
# Update the tag lists
# -----------------------------------------------------------------------------
echo "* Updating tag lists"
prof_start taglists
taglist="/tmp/taglist-${pid}"
@ -127,6 +159,7 @@ for section in "news" "events"; do
filecount["${tag}:${section}"]=$(grep "^${section}/" "${tagmaps}/${tag}" | wc --lines || true)
done
done
prof_elapsed taglists "update tag lists - filecount" >&2
for language in $(ls ${taglabels}); do
{
@ -166,6 +199,7 @@ for language in $(ls ${taglabels}); do
rm -f "${taglist}"
done
prof_elapsed taglists "update tag lists" >&2
# -----------------------------------------------------------------------------
# Remove the temporary directory
@ -179,6 +213,7 @@ rm -rf "${taglabels}"
# -----------------------------------------------------------------------------
echo "* Updating XML lists"
prof_start xmllists
all_xml="$(find * -name '*.??.xml' | sed -r 's/\...\.xml$//' | sort -u)"
@ -222,12 +257,14 @@ for source_file in $(find * -name '*.sources' | sort); do
rm -f "/tmp/xmllist-${pid}"
done
prof_elapsed xmllists "update xml lists" >&2
# -----------------------------------------------------------------------------
# Touch all .xmllist files where one of the contained files has changed
# -----------------------------------------------------------------------------
echo "* Checking contents of XML lists"
prof_start checkcontents
for list_file in $(find -name '.*.xmllist' -printf '%P\n' | sort); do
if [ ! -s "${list_file}" ]; then # Empty file list
@ -244,3 +281,4 @@ for list_file in $(find -name '.*.xmllist' -printf '%P\n' | sort); do
touch "${list_file}"
fi
done
prof_elapsed checkcontents "check xml contents" >&2

Loading…
Cancel
Save