Order and improve status pages
All checks were successful
continuous-integration/drone/pr Build is passing

This commit is contained in:
delliott 2024-08-14 11:49:14 +02:00
parent 0746034e56
commit e6dea9e42d
Signed by: delliott
SSH Key Fingerprint: SHA256:h5IC6Ec/o6ypmpM7ZdRqULGyG8dz4Jr5K192BAWAjoo
20 changed files with 441 additions and 296 deletions

6
.gitignore vendored
View File

@ -15,3 +15,9 @@ global/data/topbanner/.topbanner.??.xml
search/index.js
tags/tagged-*.en.xhtml
tags/.tags.??.xml
## Status dir stuff
status.fsfe.org/*fsfe.org/*
!status.fsfe.org/*fsfe.org/index.cgi
!status.fsfe.org/*fsfe.org/template.en.xhtml
status.fsfe.org/translations/data/*

View File

@ -261,3 +261,6 @@ RewriteRule ^about/printable(/.*)?$ /contribute/spreadtheword.html [R=301,L]
# Forbid access to these folders
RewriteRule ^(tools|build)(/.*)?$ - [F,L]
# Redirect attempts to status to status.fsfe.org
RewriteRule ^status.fsfe.org(/.*)?$ https://status.fsfe.org/ [R=301,L]

View File

@ -70,12 +70,12 @@ if [ -z "$inc_arguments" ]; then
readonly basedir="${basedir:+$(realpath "$basedir")}"
readonly domain="${domain:-www.fsfe.org}"
readonly command
if [ "$stagedir" != "$target" ] && printf %s "$target" |egrep -q '^.+@.+:(.+)?$'; then
if [ "$stagedir" != "$target" ] && printf %s "$target" | egrep -q '^.+@.+:(.+)?$'; then
readonly target
else
readonly target="${target:+$(realpath "$target")}"
fi
case "$command" in
build_into) [ -z "$target" ] && die "Missing destination directory" ;;
git_build_into) [ -z "$target" ] && die "Missing destination directory" ;;

View File

@ -47,7 +47,7 @@ if [ -n "$statusdir" ]; then
"either select a status directory that exists and is writable,\n" \
"or run the build script without output to a status directory"
fi
readonly statusdir="${statusdir:+$(realpath "$statusdir")}"
readonly statusdir="${statusdir:+$(realpath "$statusdir")}"
buildpids=$(
ps -eo command \

View File

@ -37,8 +37,6 @@ buildrun(){
ncpu="$(grep -c ^processor /proc/cpuinfo)"
[ -n "$statusdir" ] && cp "$basedir/build/status.html.sh" "$statusdir/index.cgi"
[ -f "$(logname lasterror)" ] && rm "$(logname lasterror)"
[ -f "$(logname debug)" ] && rm "$(logname debug)"

View File

@ -189,7 +189,7 @@ EOF
# -----------------------------------------------------------------------------
# All files which should just be copied over
COPY_SRC_FILES := \$(shell find "\$(INPUTDIR)" -type f \
COPY_SRC_FILES := \$(shell find -L "\$(INPUTDIR)" -type f \
-not -path '\$(INPUTDIR)/.git/*' \
-not -path '\$(INPUTDIR)/build/*' \
-not -path '\$(INPUTDIR)/global/*' \
@ -237,7 +237,7 @@ clean:
\$(file >\$(STATUSDIR)/manifest)
\$(foreach filename,\$(ALL_DST),\$(file >>\$(STATUSDIR)/manifest,\$(filename)))
sort "\$(STATUSDIR)/manifest" > "\$(STATUSDIR)/manifest.sorted"
find -L "\$(OUTPUTDIR)" -type f \\
find -L "\$(OUTPUTDIR)" -type f -path "\$(STATUSDIR)" -prune \\
| sort \\
| diff - "\$(STATUSDIR)/manifest.sorted" \\
| sed -rn 's;^< ;;p' \\

View File

@ -54,7 +54,7 @@
</xsl:element>
<xsl:choose>
<xsl:when test="$build-env = 'development'">
<xsl:when test="$build-env = 'development' and not(/buildinfo/document/@external)">
<xsl:choose>
<xsl:when test="$mode = 'valentine'">
<xsl:element name="link">

View File

@ -30,6 +30,7 @@
</xsl:attribute>
<xsl:element name="img">
<xsl:attribute name="src">
<xsl:value-of select="$urlprefix"/>
<xsl:text>/graphics/services/mastodon.png</xsl:text>
</xsl:attribute>
<xsl:attribute name="alt">Mastodon</xsl:attribute>
@ -42,6 +43,7 @@
</xsl:attribute>
<xsl:element name="img">
<xsl:attribute name="src">
<xsl:value-of select="$urlprefix"/>
<xsl:text>/graphics/services/peertube.png</xsl:text>
</xsl:attribute>
<xsl:attribute name="alt">Peertube</xsl:attribute>

View File

@ -55,22 +55,8 @@ elif [ "$duration" -gt 0 ]; then
fi)
printf %s\\n\\n "Content-Type: text/html;charset=utf-8"
sed -e '/<!--\ spacing-comment\ -->/,$d' template.en.html
cat <<-HTML_END
<!DOCTYPE HTML>
<html lang="en">
<head>
<link rel="stylesheet" href="https://fsfe.org/look/fsfe.min.css">
<link rel="icon" href="https://fsfe.org/graphics/fsfe.ico" type="image/x-icon">
<title>Build Status</title>
<style>
div.scrollbox {
max-height: 30em;
overflow: scroll;
display: inline-block;
}
</style>
</head>
<body>
<h1>Build report</h1>
<dl class="buildinfo">
<dt>Start time:</dt><dd>$(timestamp ${start_time})</dd>
@ -197,6 +183,5 @@ $(
)
</div>
</details>
</body>
</html>
HTML_END
sed -n -e '/<\/body>/,$p' template.en.html

View File

@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<html external="true">
<version>0</version>
<head>
<style>
div.scrollbox {
max-height: 30em;
width=80%;
overflow: scroll;
display: inline-block;
}
</style>
<title>Build Status</title>
</head>
<body>
<!-- spacing-comment -->
</body>
</html>

View File

@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<html external="true">
<version>0</version>
<head>
<title>Website status</title>
</head>
<body>
<h1>Website status</h1>
<p>The following status information is available:</p>
<h2>fsfe.org</h2>
<ul>
<li>
<a href="fsfe.org/index.cgi">Last build script output</a>
</li>
<li>
<a href="translations/">Translation status</a>
</li>
</ul>
<h2>test.fsfe.org</h2>
<ul>
<li>
<a href="test.fsfe.org/index.cgi">Last build script output</a>
</li>
</ul>
</body>
</html>

View File

@ -0,0 +1 @@
../fsfe.org/index.cgi

View File

@ -0,0 +1 @@
../fsfe.org/template.en.xhtml

View File

@ -0,0 +1,215 @@
#!/usr/bin/env bash
set -euo pipefail
while getopts h OPT; do
case $OPT in
h)
print_usage
exit 0
;;
*)
echo "Unknown option: -$OPTARG"
print_usage
exit 1
;;
esac
done
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
readonly SCRIPT_DIR
REPO="$SCRIPT_DIR"/../..
readonly REPO
OUT="${REPO}"/status.fsfe.org/translations/data
readonly OUT
cd "${REPO}" || exit 2
echo "Making required directories!"
mkdir -p "$OUT"
LOGFILE="${OUT}/log.txt"
langs="$(
find ./global/languages -type f -printf "%f\n" | while read -r lang; do
echo "$lang $(cat ./global/languages/"$lang")"
done
)"
readonly langs
texts_dir="global/data/texts"
readonly texts_dir
texts_en=$(grep 'id=".*"' ${texts_dir}/texts.en.xml | perl -pe 's/.*id=\"(.*?)\".*/\1/g')
readonly texts_en
# Make filedates match git commits
echo "Begin syncing filedates with git commit dates" | tee "${LOGFILE}"
"$SCRIPT_DIR"/filedate-sync-git.sh >>"${LOGFILE}"
echo "File date sync finished" | tee -a "${LOGFILE}"
files=""
# Grouped by priority
files+=$'\n'$(find ./index.en.xhtml | sed 's/$/ 1/')
files+=$'\n'$(find ./freesoftware/freesoftware.en.xhtml | sed 's/$/ 1/')
files+=$'\n'$(find ./activities -type f \( -iname "activity\.en\.xml" \) | sed 's/$/ 2/')
files+=$'\n'$(find ./activities -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 3/')
files+=$'\n'$(find ./freesoftware -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 3/')
files+=$'\n'$(find ./order -maxdepth 1 -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 4/')
files+=$'\n'$(find ./contribute -maxdepth 1 -type f \( -iname "spreadtheword*\.en\.xhtml" -o -iname "spreadtheword*\.en\.xml" \) | sed 's/$/ 4/')
files+=$'\n'$(find . -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) -mtime -200 -not -path './news/*' -not -path './events/*' | sed 's/$/ 5/')
files+=$'\n'$(find ./news -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) -mtime -30 | sed 's/$/ 5/')
# Remove files that are not in the list of those managed by git
tmp=""
files="$(
echo "$files" | while read -r line priority; do
if [[ "$(git ls-files | grep -o "${line:2}" | wc -l)" -ge 1 ]] && [[ "$(echo "$tmp" | grep "$line")" == "" ]]; then
echo "$line" "$priority"
tmp+="$line"
fi
done
)"
unset tmp
files=$(echo "$files" | grep -v "internal\|order\/data\/items\.en\.xml\|donate\/germany\|donate\/netherlands\|donate\/switzerland\|status.fsfe.org\|boilerplate\|\/\..*\.xml\|)")
readonly files
prevlang=""
prevprio=""
echo "Begin generating translation status for all languages" | tee -a "$LOGFILE"
statuses="$(
echo "$files" | while read -r fullname priority; do
ext="${fullname##*.}"
base="${fullname//\.[a-z][a-z]\.${ext}/}"
original_version=$(xsltproc build/xslt/get_version.xsl "$base".en."$ext")
echo "$langs" | while read -r lang_short lang_long; do
i="$base"."$lang_short"."$ext"
echo "Processing file $i" >>"$LOGFILE"
if [[ -f $i ]]; then
translation_version=$(xsltproc build/xslt/get_version.xsl "$i")
else
translation_version="-1"
fi
if [ "${translation_version:-0}" -lt "${original_version:-0}" ]; then
originaldate=$(git log --pretty="%cd" --date=raw -1 "$base".en."$ext" | cut -d' ' -f1)
if [ "$ext" == "xhtml" ]; then
original_url="https://webpreview.fsfe.org?uri=/${base/#\.\//}.en.html"
translation_url="https://webpreview.fsfe.org?uri=/${base/#\.\//}.$lang_short.html"
elif [ "$ext" == "xml" ]; then
original_url="https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${base/#\.\//}.en.xml"
translation_url="https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${base/#\.\//}.$lang_short.xml"
else
translation_url="#"
original_url="#"
fi
if [ "$translation_version" == "-1" ]; then
translation_url="#"
fi
echo "$lang_short $lang_long $base $priority $originaldate $original_url $original_version $translation_url ${translation_version/-1/Untranslated}"
fi
done
done | sort -t' ' -k 1,1 -k 4,4 -k 3,3 | cat - <(echo "zz zz zz zz zz zz zz zz zz")
)"
echo "Status Generated" | tee -a "$LOGFILE"
echo "Generate language status overview" | tee -a "$LOGFILE"
cat >"${OUT}/langs.en.xml" <<-EOF
<?xml version="1.0" encoding="UTF-8"?>
<translation-overall-status>
<version>1</version>
EOF
echo "$langs" | while read -r lang_short lang_long; do
declare -A prio_counts
for i in {1..6}; do
prio_counts["$i"]="<priority number=\"$i\" value=\"$(echo "$statuses" | sed -n "/^$lang_short\ $lang_long\ [^\ ]*\ $i/p" | wc -l)\"/>"
done
cat >>"${OUT}/langs.en.xml" <<-EOF
<language short="$lang_short" long="$lang_long">
${prio_counts["1"]}
${prio_counts["2"]}
</language>
EOF
unset prio_counts
done
cat >>"${OUT}/langs.en.xml" <<-EOF
</translation-overall-status>
EOF
echo "Finished Generating status overview" | tee -a "$LOGFILE"
echo "Create language pages" | tee -a "$LOGFILE"
echo "$statuses" | while read -r lang_short lang_long page priority originaldate original_url original_version translation_url translation_version; do
if [[ "$prevlang" != "$lang_short" ]]; then
if [[ "$prevlang" != "" ]]; then
# Translatable strings
texts_file="${texts_dir}/texts.${prevlang}.xml"
missing_texts=()
longest_text_length=0
for text in $texts_en; do
if ! xmllint --xpath "//text[@id = \"${text}\"]" "${texts_file}" &>/dev/null; then
missing_texts+=("$text")
tmp_length="${#text}"
if [ "$tmp_length" -gt "$longest_text_length" ]; then
longest_text_length="$tmp_length"
fi
fi
done
for index in "${!missing_texts[@]}"; do
missing_texts["$index"]="<text>${missing_texts["$index"]}</text>"$'\n'
done
cat >>"${OUT}/translations.$prevlang.xml" <<-EOF
</priority>
<missing-texts>
<url>https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${texts_file}</url>
<max-length>$((longest_text_length + 5))ch</max-length>
<filename> $texts_file</filename>
${missing_texts[*]}
</missing-texts>
</translation-status>
EOF
if [[ "$lang_short" == "zz" ]]; then
break
fi
fi
cat >"${OUT}/translations.$lang_short.xml" <<-EOF
<?xml version="1.0" encoding="UTF-8"?>
<translation-status>
<version>1</version>
EOF
prevprio=""
prevlang=$lang_short
fi
if [[ "$priority" != "$prevprio" ]]; then
if [[ "$prevprio" != "" ]]; then
cat >>"${OUT}/translations.$lang_short.xml" <<-EOF
</priority>
EOF
fi
cat >>"${OUT}/translations.$lang_short.xml" <<-EOF
<priority value="$priority">
EOF
prevprio=$priority
fi
orig=$(date +"%Y-%m-%d" --date="@$originaldate")
if [[ $originaldate -gt $(date +%s --date='6 months ago') ]]; then
new=true
else
new=false
fi
cat >>"${OUT}/translations.$lang_short.xml" <<-EOF
<file>
<page>$page</page>
<original_date>$orig</original_date>
<new>$new</new>
<original_url>$original_url</original_url>
<original_version>$original_version</original_version>
<translation_url>$translation_url</translation_url>
<translation_version>$translation_version</translation_version>
</file>
EOF
done
echo "Finished creating language pages" | tee -a "$LOGFILE"
echo "Finished !"
# rm "$LOGFILE"

View File

@ -0,0 +1,126 @@
<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
<xsl:import href="../../fsfe.xsl"/>
<xsl:template match="translation-overall-status">
<details>
<summary>
Overall Translation Status Overview
</summary>
<table>
<tr>
<th>Language</th>
<th>Priority 1 files in need of translation</th>
<th>Priority 2 files in need of translation</th>
</tr>
<xsl:for-each select="/buildinfo/document/set/language">
<tr>
<td>
<a>
<xsl:attribute name="href">
<xsl:text>index.</xsl:text>
<xsl:value-of select="@short"/>
<xsl:text>.html</xsl:text>
</xsl:attribute>
<xsl:value-of select="@long"/>
</a>
</td>
<xsl:for-each select="priority">
<xsl:sort select="@number" order="ascending"/>
<td>
<xsl:value-of select="@value"/>
</td>
</xsl:for-each>
</tr>
</xsl:for-each>
</table>
</details>
</xsl:template>
<xsl:template match="translation-status">
<xsl:if test="/buildinfo/@language != 'en'">
<div class="translation-status">
<h2>
<xsl:text>Translation-Status for </xsl:text>
<xsl:value-of select="/buildinfo/@language"/>
</h2>
<h3>File translations</h3>
<xsl:for-each select="/buildinfo/document/set/priority">
<xsl:sort select="@value" order="ascending"/>
<details>
<summary>
<xsl:text>Priority: </xsl:text>
<xsl:value-of select="@value"/>
</summary>
<table>
<tr>
<th>Page</th>
<th>Original date</th>
<th>Original version</th>
<th>Translation version</th>
</tr>
<xsl:for-each select="file">
<tr>
<td>
<xsl:value-of select="page"/>
</td>
<td>
<xsl:value-of select="original_date"/>
</td>
<td>
<a>
<xsl:attribute name="href">
<xsl:value-of select="original_url"/>
</xsl:attribute>
<xsl:value-of select="original_version"/>
</a>
</td>
<td>
<xsl:choose>
<xsl:when test="translation_url != '#'">
<a>
<xsl:attribute name="href">
<xsl:value-of select="translation_url"/>
</xsl:attribute>
<xsl:value-of select="translation_version"/>
</a>
</xsl:when>
<xsl:otherwise>
<xsl:value-of select="translation_version"/>
</xsl:otherwise>
</xsl:choose>
</td>
</tr>
</xsl:for-each>
</table>
</details>
</xsl:for-each>
<xsl:for-each select="/buildinfo/document/set/missing-texts">
<xsl:variable name="max-length" select="max-length"/>
<h3>
<xsl:text>Missing texts in </xsl:text>
<a>
<xsl:attribute name="href">
<xsl:value-of select="url"/>
</xsl:attribute>
<xsl:value-of select="filename"/>
</a>
</h3>
<details>
<summary>Show</summary>
<div style="display: flex; flex-wrap: wrap;">
<xsl:for-each select="text">
<div>
<xsl:attribute name="style">
<xsl:text>width: </xsl:text>
<xsl:value-of select="$max-length"/>
<xsl:text>;</xsl:text>
</xsl:attribute>
<xsl:value-of select="current()"/>
</div>
</xsl:for-each>
</div>
</details>
</xsl:for-each>
</div>
</xsl:if>
</xsl:template>
</xsl:stylesheet>

View File

@ -0,0 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<html external="true">
<version>1</version>
<head>
<title>FSFE Translation Status</title>
</head>
<body>
<h1>Translation Status</h1>
<h2>Overall Status</h2>
<translation-overall-status/>
<h2>Translation Information</h2>
<p><span style="color: red">Red entries</span> are pages where the original is newer than 6 months.
</p>
<p>Priority decreases with higher numbers. Eg. 1 is highest priority.</p>
<p>Simply click on a priority or the text section to expand it and see contained information</p>
<p>Now, a brief explanation of each column</p>
<dl>
<dt>Page</dt>
<dd>The filepath of the page, relative to website root</dd>
<dt>Original Date</dt>
<dd>Date the original file (Engligh version) was modified</dd>
<dt>Original Version</dt>
<dd>The version number of the original file. Is also a link to the original file, viewable with the webpreview tool if possible and just the raw source from the website repo if not.</dd>
<dt>Translation Version</dt>
<dd>The version number of the translated file. Is also a link to the translated file, viewable with the webpreview tool if possible and just the raw source from the website repo if not. If the file has not been translated then the link simply goes nowhere.</dd>
</dl>
<translation-status/>
</body>
</html>

View File

@ -0,0 +1 @@
status.fsfe.org/translations/data/*:[]

View File

@ -205,7 +205,7 @@ for f in $files_all; do
fileregex="(\.xhtml$|\.xml$|\.xsl$)"
if matchfile "${f}" "${fileregex}"; then
regex="(utf-8|us-ascii)"
if ! [[ $(file -b --mime-encoding "${f}") =~ $regex ]]; then
if ! [[ $(file -b --mime-encoding "$(realpath "${f}")") =~ $regex ]]; then
RETURN_ENC=$((RETURN_ENC + 1))
FILES_ENC="${FILES_ENC}|${f}"
fi

View File

@ -1,270 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
print_usage() { echo "make-outdated-list.sh -o outdir -r repo_path"; }
while getopts o:r:h OPT; do
case $OPT in
o) OUT=$OPTARG ;;
r) REPO=$OPTARG ;;
h)
print_usage
exit 0
;;
*)
echo "Unknown option: -$OPTARG"
print_usage
exit 1
;;
esac
done
if [[ -z "${OUT}" || -z "${REPO}" ]]; then
echo "Mandatory option missing:"
print_usage
exit 1
fi
cd "${REPO}" || exit 2
prevlang=''
prev_priority=''
texts_dir="global/data/texts"
texts_en=$(grep 'id=".*"' ${texts_dir}/texts.en.xml | perl -pe 's/.*id=\"(.*?)\".*/\1/g')
OUT_TMP="${OUT}/.tmp-translations"
LOGFILE="${OUT}/log.txt"
echo "Making required dirs." | tee "$LOGFILE"
mkdir -p "${OUT_TMP}/translations" || exit
echo "Making index" | tee -a "$LOGFILE"
cat >"${OUT_TMP}/translations.html" <<-EOF
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="https://fsfe.org/look/fsfe.min.css">
<link rel="icon" href="https://fsfe.org/graphics/fsfe.ico" type="image/x-icon">
<title>FSFE Translation Languages</title>
</head>
<h1>Translation Status Page</h1>
<body>
<p>Click on the links below to jump to a particular language</p>
EOF
: >"${OUT_TMP}/translations/langs.txt"
find . -type f -iname "*\.en\.xhtml" | sed 's/\.[a-z][a-z]\.xhtml//' | sort | while read -r A; do
for i in "$A".[a-z][a-z].xhtml; do
# shellcheck disable=SC2001
lang_uniq=$(echo "$i" | sed 's/.*\.\([a-z][a-z]\)\.xhtml/\1/')
if [ "$lang_uniq" != "en" ]; then
echo "$lang_uniq"
fi
done
done | sort | uniq | while read -r lang_uniq; do
cat >>"${OUT_TMP}/translations.html" <<-EOF
<a href="/translations/$lang_uniq.html">$lang_uniq</a>
EOF
cat >>"${OUT_TMP}/translations/langs.txt" <<-EOF
$lang_uniq
EOF
done
cat >>"${OUT_TMP}/translations.html" <<-EOF
</body>
</html>
EOF
echo "Index finished" | tee -a "$LOGFILE"
# Make filedates match git commits
echo "Begin syncing filedates with git commit dates" | tee -a "${LOGFILE}"
./tools/filedate-sync-git.sh >>"${LOGFILE}"
echo "File date sync finished" | tee -a "${LOGFILE}"
files=""
# Grouped by priority
files+=$'\n'$(find ./index.en.xhtml | sed 's/$/ 1/')
files+=$'\n'$(find ./freesoftware/freesoftware.en.xhtml | sed 's/$/ 1/')
files+=$'\n'$(find ./activities -type f \( -iname "activity\.en\.xml" \) | sed 's/$/ 2/')
files+=$'\n'$(find ./activities -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 3/')
files+=$'\n'$(find ./freesoftware -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 3/')
files+=$'\n'$(find ./order -maxdepth 1 -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) | sed 's/$/ 4/')
files+=$'\n'$(find ./contribute -maxdepth 1 -type f \( -iname "spreadtheword*\.en\.xhtml" -o -iname "spreadtheword*\.en\.xml" \) | sed 's/$/ 4/')
files+=$'\n'$(find . -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) -mtime -365 -not -path './news/*' -not -path './events/*' | sed 's/$/ 5/')
files+=$'\n'$(find ./news -type f \( -iname "*\.en\.xhtml" -o -iname "*\.en\.xml" \) -mtime -30 | sed 's/$/ 5/')
# Remove files that are not in the list of those managed by git
tmp=""
files="$(echo "$files" | while read -r line priority; do if [[ "$(git ls-files | grep -o "${line:2}" | wc -l)" -ge 1 ]] && [[ "$(echo "$tmp" | grep "$line")" == "" ]]; then
echo "$line" "$priority"
tmp+="$line"
fi; done)"
# List of dirs/files to exclude
exclude=(
# Internal pages
"internal"
# Order Items
"order/data/items.en.xml"
# Stuff not really important for other languages
"donate/germany"
"donate/netherlands"
"donate/switzerland"
# The boilerplate
"boilerplate"
# Some misc xml files
"/\\\..*\\\.xml"
)
exclude_string=""
exclude_indexes=("${!exclude[@]}")
last_index=${exclude_indexes[-1]}
for index in "${!exclude[@]}"; do
exclude["$index"]="$(echo "${exclude["$index"]}" | xargs | sed 's/\//\\\//g')"
if [[ "$index" != "$last_index" ]]; then
exclude_string+="${exclude["$index"]}\|"
else
exclude_string+="${exclude["$index"]}"
fi
done
files=$(echo "$files" | grep -v "$exclude_string" | sort)
echo "Begin generating language status for found langs" | tee -a "$LOGFILE"
echo "$files" | while read -r fullname priority; do
ext="${fullname##*.}"
base="${fullname//\.[a-z][a-z]\.${ext}/}"
original_version=$(xsltproc build/xslt/get_version.xsl "$base".en."$ext")
while read -r lang; do
i="$base"."$lang"."$ext"
echo "Processing file $i" >>"$LOGFILE"
if [[ -f $i ]]; then
translation_version=$(xsltproc build/xslt/get_version.xsl "$i")
else
translation_version="-1"
fi
if [ "${translation_version:-0}" -lt "${original_version:-0}" ]; then
originaldate=$(git log --pretty="%cd" --date=raw -1 "$base".en."$ext" | cut -d' ' -f1)
# shellcheck disable=SC2001
lang=$(echo "$i" | sed "s/.*\.\([a-z][a-z]\)\.$ext/\1/")
if [ "$ext" == "xhtml" ]; then
original_url="https://webpreview.fsfe.org?uri=/${base/#\.\//}.en.html"
translation_url="https://webpreview.fsfe.org?uri=/${base/#\.\//}.$lang.html"
elif [ "$ext" == "xml" ]; then
original_url="https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${base/#\.\//}.en.xml"
translation_url="https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${base/#\.\//}.$lang.xml"
else
translation_url="#"
original_url="#"
fi
if [ "$translation_version" == "-1" ]; then
translation_url="#"
fi
echo "$lang $base $priority $originaldate $original_url $original_version $translation_url ${translation_version/-1/Untranslated}"
fi
done <"${OUT_TMP}/translations/langs.txt"
done | sort -t' ' -k 1,1 -k 3,3 -k 2,2 |
while read -r lang page priority originaldate original_url original_version translation_url translation_version; do
if [[ "$prevlang" != "$lang" ]]; then
if [[ "$prevlang" != "" ]]; then
cat >>"${OUT_TMP}/translations/$prevlang.html" <<-EOF
</table>
</details>
EOF
# Translatable strings
texts_file="${texts_dir}/texts.${prevlang}.xml"
missing_texts=()
longest_text_length=0
for text in $texts_en; do
if ! xmllint --xpath "//text[@id = \"${text}\"]" "${texts_file}" &>/dev/null; then
missing_texts+=("$text")
tmp_length="${#text}"
if [ "$tmp_length" -gt "$longest_text_length" ]; then
longest_text_length="$tmp_length"
fi
fi
done
for index in "${!missing_texts[@]}"; do
missing_texts["$index"]="<div style=\"width: $((longest_text_length + 5))ch;\">${missing_texts["$index"]}</div>"$'\n'
done
cat >>"${OUT_TMP}/translations/$prevlang.html" <<-EOF
<p>
Missing texts in <a href="https://git.fsfe.org/FSFE/fsfe-website/src/branch/master/${texts_file}">${texts_file}</a>:
</p>
<details>
<summary>
Show missing texts
</summary>
<div style="width: 100%; display: flex; flex-wrap: wrap;">
${missing_texts[*]}
</div>
</details>
</body>
</html>
EOF
fi
cat >"${OUT_TMP}/translations/$lang.html" <<-EOF
<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<link rel="stylesheet" href="https://fsfe.org/look/fsfe.min.css">
<link rel="icon" href="https://fsfe.org/graphics/fsfe.ico" type="image/x-icon">
<title>FSFE Translation Status: $lang</title>
</head>
<body>
<h1 id="$lang">Language: $lang</h1>
<p><span style="color: red">Red entries</span> are pages where the original is newer than 6 months.</p>
<p>Priority decreases with higher numbers. Eg. 1 is highest priority.</p>
<p>Simply click on a priority or the text section to expand it and see contained information</p>
<p>Now, a brief explanation of each column</p>
<dl>
<dt>Page</dt>
<dd>The filepath of the page, relative to website root</dd>
<dt>Original Date</dt>
<dd>Date the original file (Engligh version) was modified</dd>
<dt>Original Version</dt>
<dd>The version number of the original file. Is also a link to the original file, viewable with the webpreview tool if possible and just the raw source from the website repo if not.</dd>
<dt>Translation Version</dt>
<dd>The version number of the translated file. Is also a link to the translated file, viewable with the webpreview tool if possible and just the raw source from the website repo if not. If the file has not been translated then the link simply goes nowhere.</dd>
</dl>
EOF
prev_priority=""
prevlang=$lang
fi
if [[ "$priority" != "$prev_priority" ]]; then
if [[ "$prev_priority" != "" ]]; then
cat >>"${OUT_TMP}/translations/$lang.html" <<-EOF
</table>
</details>
EOF
fi
cat >>"${OUT_TMP}/translations/$lang.html" <<-EOF
<details>
<summary>
Priority: $priority
</summary>
<table>
<tr><th>Page</th><th>Original date</th><th>Original version</th><th>Translation version</th></tr>
EOF
prev_priority=$priority
fi
orig=$(date +"%Y-%m-%d" --date="@$originaldate")
if [[ $originaldate -gt $(date +%s --date='6 months ago') ]]; then
color='color: red;'
else
color=''
fi
cat >>"${OUT_TMP}/translations/$lang.html" <<-EOF
<tr><td><a style="$color">$page</a></td><td>$orig</td><td><a href="$original_url">$original_version</a></td><td><a href="$translation_url">$translation_version</a></td></tr>
EOF
done
echo "Finished creating language pages" | tee -a "$LOGFILE"
echo "Replacing old output" | tee -a "$LOGFILE"
rsync -a --remove-source-files "${OUT_TMP}"/ "$OUT" | tee -a "$LOGFILE"
echo "Finished" | tee -a "$LOGFILE"
rm "$LOGFILE"