improved logging, some relieve to caching artifacts in dependency build, still to much globbing

svn path=/branches/test/; revision=31575

build/arguments.sh

@@ -69,7 +69,7 @@ if [ -z "$inc_arguments" ]; then
         [ "$#" -gt 0 ] && shift 1 && globfile="$1"
         ;;
       *)
-        print_error "Unknown option $1"
+        print_error "Unknown option: $1"
         exit 1
         ;;
     esac

build/build_main.sh

@@ -20,7 +20,7 @@ buildpids=$(
   | wc -l
 )
 if [ $command = "build_into" -o $command = "svn_build_into" ] && [ "$buildpids" -gt 2 ]; then
-  print_error "build script is already running"
+  debug "build script is already running"
   exit 0
 fi
 
@@ -32,8 +32,6 @@ fi
 [ -z "$inc_scaffold" ]  && . "$basedir/build/scaffold.sh"
 [ -z "$inc_sources" ]   && . "$basedir/build/sources.sh"
 
-[ -s "$(logname debug)" ] && truncate -s 0 "$(logname debug)"
-
 case "$command" in
   map_tags)        map_tags "$@";;
   build_into)      build_into ;;

build/buildrun.sh

@@ -9,8 +9,11 @@ inc_buildrun=true
 build_into(){
   ncpu="$(grep -c ^processor /proc/cpuinfo)"
 
+  [ -s "$(logname debug)" ] && truncate -s 0 "$(logname debug)"
   forcelog Makefile
 
+  validate_caches
+
   make -j $ncpu -C "$basedir" \
   | logstatus premake
 
@@ -38,18 +41,17 @@ svn_build_into(){
   svn --non-interactive update "$basedir" >"$SVNchanges" 2>"$SVNerrors"
 
   if [ -s "$SVNerrors" ]; then
-    print_error "SVN reported the following problem:\n" \
-                "$(cat "$SVNerrors")"
-    exit 1
+    die "SVN reported the following problem:\n" \
+        "$(cat "$SVNerrors")"
   elif egrep '^(C...|.C..|...C) .+' "$SVNchanges"; then
-    print_error "SVN encountered a conflict:\n" \
-                "$(cat "$SVNchanges")"
-    exit 1
+    die "SVN encountered a conflict:\n" \
+        "$(cat "$SVNchanges")"
   elif egrep '^At revision [0-9]+\.' "$SVNchanges"; then
     debug "No changes to SVN:\n" \
           "$(cat "$SVNchanges")"
     exit 0
   else
+    logstatus SVNlatest <"$SVNchanges"
     regen_globs=false
     regen_xhtml=false
     regen_xsldeps=false

build/logging.sh

@@ -27,3 +27,8 @@ logstatus(){
   # pipeline atom to write data streams into a log file
   tee "$(logname "$1")"
 }
+
+logappend(){
+  # pipeline atom to write data streams into a log file
+  tee -a "$(logname "$1")"
+}

build/makerules.sh

@@ -41,7 +41,7 @@ glob_maker(){
   sourceglobfile="${filedir%/.}/._._${shortbase}.sourceglobs"
 
   cat <<MakeEND
-$(mes "$sourceglobfile"): $(mes $(all_sources "$input/$sourcesfile")) \${INPUTDIR}/tagmap
+$(mes "$sourceglobfile"): \${INPUTDIR}/tagmap
 	\${PGLOBBER} \${PROCFLAGS} sourceglobs "\${INPUTDIR}/$sourcesfile" >"$sourceglobfile"
 MakeEND
 

build/misc.sh

@@ -8,25 +8,23 @@ print_help(){
 }
 
 match(){
-  echo -E "$1" |egrep -q "$2"
+  printf %s "$1" |egrep -q "$2"
 }
 
 debug(){
-  dbg_file="$(logname debug)"
-
   if [ "$#" -ge 1 ]; then
-    echo "$@" |tee -a "$dbg_file"
+    echo "$(date '+%F %T'): $@" |logappend debug
   else
-    tee -a "$dbg_file"
+    logappend debug
   fi
 }
 
 print_error(){
-  echo "Error: $@" |logstatus lasterror >&2
+  echo "Error - $@" |logappend lasterror >&2
   echo "Run '$0 --help' to see usage instructions" >&2
 }
 
 die(){
-  print_error "$@"
+  echo "$(date '+%F %T'): Fatal - $@" |logappend lasterror >&2
   exit 1
 }

build/sources.sh

@@ -3,6 +3,13 @@
 inc_sources=true
 [ -z "$inc_xmlfiles" ] && . "$basedir/build/xmlfiles.sh"
 
+validate_tagmap(){
+  tagmap="$basedir/tagmap"
+  sed -rn 's;^(.*\.xml) +.*$;\1;p' "$tagmap" |while read fn; do
+    [ -f "$fn" ] || touch -cd@0 "$tagmap"
+  done
+}
+
 map_tags(){
   for xml in "$@"; do
     printf '%s ' "$xml"
@@ -80,19 +87,6 @@ sourceglobs(){
   fi
 }
 
-all_sources(){
-  # read a .sources file and glob up all referenced
-  # source files
-  sourcesfile="$1"
-
-  if [ -f "$sourcesfile" ]; then
-    sed -rn 's;:global$;*.[a-z][a-z].xml;gp' "$sourcesfile" \
-    | while read glob; do
-      echo "$basedir/"$glob
-    done |grep -vF "[a-z][a-z].xml"
-  fi
-}
-
 list_sources(){
   # read a .sources file and generate a list
   # of all referenced xml files with preference
@@ -138,12 +132,11 @@ cast_globfile(){
 
   sources="$(list_sources "###" "$lang" "$(cat "$sourceglobfile")")"
 
-  [ -f "$globfile" ] && \
-    [ "$sources" = "$(cat "$globfile")" ] \
-  || echo "$sources" >"$globfile"
+  [ -f "$globfile" ] && [ "$sources" = "$(cat "$globfile")" ] \
+  || printf %s "$sources" >"$globfile"
 
   if [ "$sourceglobfile" -nt "$globfile" ]; then
-    echo "$sources" |while read incfile; do
+    printf %s "$sources" |while read incfile; do
       [ "$incfile" -nt "$globfile" ] && touch "$globfile" || true
     done
   fi

build/stirrups.sh

@@ -2,6 +2,14 @@
 
 inc_stirrups=true
 [ -z "$inc_misc" ] && . "$basedir/build/misc.sh"
+[ -z "$inc_sources" ] && . "$basedir/build/sources.sh"
+
+validate_caches(){
+  # outdate / remove cache files if necessary
+  # hook functions here as required
+
+  validate_tagmap  # hook from sources.sh
+}
 
 dir_maker(){
   # set up directory tree for output
@@ -40,7 +48,9 @@ remove_orphans(){
   # Concatenate all lines from manifest and `find`.
   # Every file which is listed twice is wanted and exists.
   # We use 'uniq -u' to drop those from the list.
-  # Remaining single files exist only in the tree and are to be removed
+  # Remaining single files exist only in the tree and are to be
+  # removed (or were just added to the manifest and cannot be removed
+  # from the tree)
 
   (find "$dtree" \( -type f -o -type l \) -printf '%P\n' ; cat) \
   | sort \