refactor/search (#5588)

- save search index as json instead of js
- optimise the search index generation script

Co-authored-by: Darragh Elliott <me@delliott.net>
Reviewed-on: #5588
Co-authored-by: delliott <delliott@fsfe.org>
Co-committed-by: delliott <delliott@fsfe.org>

fsfe.org/.gitignore

@@ -9,7 +9,7 @@ news/????/index.sources
 news/????/index.xsl
 news/**/.*.??.xml
 # search index
-search/index.js
+search/index.json
 #tags
 tags/tagged-*.??.xhtml
 tags/.tags.??.xml

fsfe.org/search/search.en.xhtml

@@ -6,7 +6,6 @@
   <title>Search</title>
 
   <script type="text/javascript" src="/scripts/thirdparty/lunr.min.js"/>
-  <script type="text/javascript" src="index.js"/>
 </head>
 <body class="toplevel">
 

fsfe.org/search/search.js

@@ -21,112 +21,113 @@ through which recipients can access the Corresponding Source.
 @licend  The above is the entire license notice
 for the JavaScript code in this page.
 */
-fetch(`strings.${document.documentElement.lang}.js`)
-  .then((response) => {
+Promise.all([
+  fetch(`index.json`).then((response) => {
     if (response.ok) {
-      return document.documentElement.lang;
+      return response.json();
     } else {
-      return "en";
+      throw new Error(`Failed to fetch index.json: ${response.status}`);
     }
-  })
-  .then((language) => {
-    const text_module = `./strings.${language}.js`;
-    const importPromise = import(text_module);
+  }),
+  fetch(`strings.${document.documentElement.lang}.js`).then((response) =>
+    response.ok ? document.documentElement.lang : "en",
+  ),
+]).then(([pages, language]) => {
+  const text_module = `./strings.${language}.js`;
+  const importPromise = import(text_module);
 
-    const searchString = new URLSearchParams(window.location.search).get("q");
-    const locals = [document.documentElement.getAttribute("lang")];
-    if (!locals.includes("en")) {
-      locals.push("en");
-    }
-    const $target = document.querySelector("#search_results");
+  const searchString = new URLSearchParams(window.location.search).get("q");
+  const locals = [document.documentElement.getAttribute("lang")];
+  if (!locals.includes("en")) {
+    locals.push("en");
+  }
+  const $target = document.querySelector("#search_results");
 
-    importPromise.then((texts) => {
-      if (searchString) {
-        // Populate the field with any existing search string
-        document.querySelector("#search").value = searchString;
+  importPromise.then((texts) => {
+    if (searchString) {
+      // Populate the field with any existing search string
+      document.querySelector("#search").value = searchString;
 
-        // Our index uses title as a key of the hashmap
-        const pagesByURL = pages.reduce((acc, curr) => {
-          acc[curr.url] = curr;
-          return acc;
-        }, {});
+      // Our index uses title as a key of the hashmap
+      const pagesByURL = pages.reduce((acc, curr) => {
+        acc[curr.url] = curr;
+        return acc;
+      }, {});
 
-        index = lunr(function () {
-          this.pipeline.remove(lunr.stopWordFilter);
-          this.pipeline.remove(lunr.trimmer);
-          this.field("title", { boost: 10 });
-          this.field("tags", { boost: 5 });
-          this.field("teaser");
-          this.field("type");
-          this.ref("url");
+      index = lunr(function () {
+        this.pipeline.remove(lunr.stopWordFilter);
+        this.pipeline.remove(lunr.trimmer);
+        this.field("title", { boost: 10 });
+        this.field("tags", { boost: 5 });
+        this.field("teaser");
+        this.field("type");
+        this.ref("url");
 
-          pages.forEach(function (page) {
-            this.add(page);
-          }, this);
-        });
+        pages.forEach(function (page) {
+          this.add(page);
+        }, this);
+      });
 
-        // Do the search and filter out results not from the current local or English
-        let matches = index
-          .search(searchString)
-          .filter((p) =>
-            locals.some((local) => p.ref.includes(local + ".html")),
-          );
-
-        function display_result(matches) {
-          // workaround xsl XML tag parsing madness
-          return (
-            "<ul>" +
-            matches
-              .map((p) => {
-                title = pagesByURL[p.ref].title;
-                date = pagesByURL[p.ref].date;
-                if (date) {
-                  return (
-                    "<li>" +
-                    '<a href="' +
-                    p.ref +
-                    '">' +
-                    title +
-                    "</a>" +
-                    " (" +
-                    date +
-                    ")</li>"
-                  );
-                } else {
-                  return '<li><a href="' + p.ref + '">' + title + " </a></li>";
-                }
-              })
-              .join("") +
-            "</ul>"
+      // Do the search and filter out results not from the current local or English
+      let matches = index
+        .search(searchString)
+        .filter((p) => locals.some((local) => p.ref.includes(local + ".html")));
+
+      function display_result(matches) {
+        // workaround xsl XML tag parsing madness
+        return (
+          "<ul>" +
+          matches
+            .map((p) => {
+              title = pagesByURL[p.ref].title;
+              date = pagesByURL[p.ref].date;
+              if (date) {
+                return (
+                  "<li>" +
+                  '<a href="' +
+                  p.ref +
+                  '">' +
+                  title +
+                  "</a>" +
+                  " (" +
+                  date +
+                  ")</li>"
+                );
+              } else {
+                return '<li><a href="' + p.ref + '">' + title + " </a></li>";
+              }
+            })
+            .join("") +
+          "</ul>"
+        );
+      }
+
+      if (matches.length > 0) {
+        let [news, pages] = matches.reduce(
+          ([true_arr, false_arr], m) => {
+            if (m.ref.includes("news") === false)
+              // return true_arr and append m to false_arr
+              return [true_arr, [...false_arr, m]];
+            else return [[...true_arr, m], false_arr];
+          },
+          [[], []],
+        );
+        if (news.length > 0) {
+          news = news.sort(
+            (a, b) => pagesByURL[a.ref].date < pagesByURL[b.ref].date,
           );
+          news = news.slice(0, 15);
+          $target.innerHTML = texts.news_text + display_result(news);
         }
-
-        if (matches.length > 0) {
-          let [news, pages] = matches.reduce(
-            ([true_arr, false_arr], m) => {
-              if (m.ref.includes("news") === false)
-                // return true_arr and append m to false_arr
-                return [true_arr, [...false_arr, m]];
-              else return [[...true_arr, m], false_arr];
-            },
-            [[], []],
-          );
-          if (news.length > 0) {
-            news = news.sort(
-              (a, b) => pagesByURL[a.ref].date < pagesByURL[b.ref].date,
-            );
-            news = news.slice(0, 15);
-            $target.innerHTML = texts.news_text + display_result(news);
-          }
-          if (pages.length > 0) {
-            pages = pages.slice(0, 15);
-            $target.innerHTML += texts.pages_text + display_result(pages);
-          }
-        } else {
-          $target.innerHTML = texts.no_results_text;
+        if (pages.length > 0) {
+          pages = pages.slice(0, 15);
+          $target.innerHTML += texts.pages_text + display_result(pages);
         }
       } else {
-        $target.innerHTML = texts.empty_query_text;
+        $target.innerHTML = texts.no_results_text;
       }
-    });
+    } else {
+      $target.innerHTML = texts.empty_query_text;
+    }
   });
+});

fsfe.org/search/search.nl.xhtml

@@ -4,7 +4,6 @@
 <head>
   <title>Zoek</title>
   <script type="text/javascript" src="/scripts/thirdparty/lunr.min.js"/>
-  <script type="text/javascript" src="index.js"/>
 </head>
 <body class="toplevel">
   <h1>Zoek</h1>

fsfe.org/search/subdir.py

@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, TypedDict
 import iso639
 import nltk  # pyright: ignore[reportMissingTypeStubs]
 from fsfe_website_build.globals import CACHE_DIR
-from fsfe_website_build.lib.misc import update_if_changed
+from fsfe_website_build.lib.misc import lang_from_filename, update_if_changed
 from lxml import etree
 from nltk.corpus import (  # pyright: ignore[reportMissingTypeStubs]
     stopwords as nltk_stopwords,  # pyright: ignore[reportMissingTypeStubs]
@@ -42,9 +42,11 @@ def _find_teaser(document: etree.ElementTree) -> str:
     :returns: The text of the teaser or an empty string
     """
     trivial_length = 10
-    for p in document.xpath("//body//p"):
-        if p.text and len(p.text.strip().split(" ")) > trivial_length:
-            return p.text
+    for paragraph_text in (
+        p.text.strip() for p in document.xpath("//body//p") if p.text
+    ):
+        if len(paragraph_text.split(" ")) > trivial_length:
+            return paragraph_text
     return ""
 
 
@@ -52,30 +54,24 @@ def _process_file(file: Path, stopwords: set[str]) -> _SearchIndexEntry:
     """Generate the search index entry for a given file and set of stopwords."""
     xslt_root = etree.parse(file)
     tags = sorted(
-        str(tag.get("key"))
+        key
         for tag in xslt_root.xpath("//tag")
-        if tag.get("key") != "front-page"
+        if (key := str(tag.get("key"))) != "front-page"
     )
     return _SearchIndexEntry(
         url=f"/{file.with_suffix('.html').relative_to(file.parents[-2])}",
         tags=" ".join(tags),
         title=(
-            xslt_root.xpath("//html//title")[0].text
-            if xslt_root.xpath("//html//title")
+            titles[0].text.strip()
+            if (titles := xslt_root.xpath("//html//title"))
             else ""
         ),
         teaser=" ".join(
-            w
-            for w in sorted(_find_teaser(xslt_root).strip().split(" "))
-            if w.lower() not in stopwords
+            w for w in _find_teaser(xslt_root).split(" ") if w.lower() not in stopwords
         ),
         type="news" if "news/" in str(file) else "page",
         # Get the date of the file if it has one
-        date=(
-            xslt_root.xpath("//news[@newsdate]").get("newsdate")
-            if xslt_root.xpath("//news[@newsdate]")
-            else None
-        ),
+        date=(newsdate if (newsdate := xslt_root.xpath("//news/@newsdate")) else None),
     )
 
 
@@ -107,22 +103,22 @@ def run(source: Path, languages: list[str], processes: int, working_dir: Path) -
                 (
                     set(
                         nltk_stopwords.words(  # pyright: ignore [(reportUnknownMemberType)]
-                            iso639.Language.from_part1(
-                                file.suffixes[0].removeprefix("."),
-                            ).name.lower(),
+                            lang_full,
                         ),
                     )
-                    if iso639.Language.from_part1(
-                        file.suffixes[0].removeprefix("."),
-                    ).name.lower()
+                    if (
+                        lang_full := (
+                            iso639.Language.from_part1(
+                                lang_from_filename(file)
+                            ).name.lower()
+                        )
+                    )
                     in nltk_stopwords.fileids()  # pyright: ignore [(reportUnknownMemberType)]
                     else set()
                 ),
             )
-            for file in filter(
-                lambda file: file.suffixes[0].removeprefix(".") in languages,
-                source_dir.glob("**/*.??.xhtml"),
-            )
+            for file in source_dir.glob("**/*.??.xhtml")
+            if file.suffixes[0].removeprefix(".") in languages
         )
 
         articles = sorted(
@@ -131,6 +127,6 @@ def run(source: Path, languages: list[str], processes: int, working_dir: Path) -
         )
 
         update_if_changed(
-            working_dir.joinpath("index.js"),
-            "var pages = " + json.dumps(articles, ensure_ascii=False),
+            working_dir.joinpath("index.json"),
+            json.dumps(articles, ensure_ascii=False),
         )