Add a search functionality (Fixes #739) (#1635)
All checks were successful
continuous-integration/drone/push Build is passing

Co-authored-by: max.mehl <max.mehl@fsfe.org>
Co-authored-by: Vincent Lequertier <vincent@fsfe.org>

Reviewed-on: #1635
This commit is contained in:
vincent 2020-11-12 15:17:43 +00:00 committed by Max Mehl
parent d21250354a
commit e753e02fce
12 changed files with 400 additions and 2 deletions

1
.gitignore vendored
View File

@ -12,5 +12,6 @@ global/data/texts/.texts.??.xml
.default.xsl
.localmenu.*.xml
.*.xmllist
search/index.js
tags/tagged-*.en.xhtml
tags/.tags.??.xml

View File

@ -13,6 +13,19 @@
# This will be overwritten in the command line running this Makefile.
build_env = development
# -----------------------------------------------------------------------------
# Build search index
# -----------------------------------------------------------------------------
# This step runs a Python tool that creates an index of all news and
# articles. It extracts titles, teaser, tags, dates and potentially more.
# The result will be fed into a JS file.
.PHONY: searchindex
all: searchindex
searchindex:
python3 tools/index-website.py
# -----------------------------------------------------------------------------
# Update CSS files
# -----------------------------------------------------------------------------

View File

@ -43,6 +43,11 @@
<td><a href="http://www.gnu.org/licenses/gpl-3.0.html">GPL-3.0-or-later</a></td>
<td><a href="/scripts/filter-teams.js">Filter Teams</a></td>
</tr>
<tr>
<td><a href="/scripts/lunr-2.3.9.min.js">scripts/lunr-2.3.9.min.js</a></td>
<td><a href="https://opensource.org/licenses/MIT">MIT</a></td>
<td><a href="https://github.com/olivernn/lunr.js/releases/tag/v2.3.9">lunr-2.3.9.min.js</a></td>
</tr>
</table>
<h3>Whom to contact</h3>

View File

@ -22,7 +22,7 @@ check_dependencies() {
}
# Check dependencies for all kinds of build envs (e.g. development, fsfe.org)
check_dependencies realpath rsync xsltproc xmllint sed find egrep grep wc make tee date iconv wget shuf
check_dependencies realpath rsync xsltproc xmllint sed find egrep grep wc make tee date iconv wget shuf python3
if ! make --version | grep -q "GNU Make 4"; then
echo "The build script requires GNU Make 4.x"

View File

@ -146,6 +146,37 @@
<xsl:call-template name="fsfe-gettext"><xsl:with-param name="id" select="'change-lang'" /></xsl:call-template>
</xsl:element>
</xsl:element>
<!-- Search box -->
<xsl:element name="li">
<xsl:attribute name="id">menu-search-box</xsl:attribute>
<xsl:element name="form">
<xsl:attribute name="method">GET</xsl:attribute>
<xsl:attribute name="action">/search/search.en.html</xsl:attribute>
<xsl:element name="div">
<xsl:attribute name="class">input-group</xsl:attribute>
<xsl:element name="div">
<xsl:attribute name="class">input-group-btn</xsl:attribute>
<xsl:element name="button">
<xsl:attribute name="class">btn btn-primary</xsl:attribute>
<xsl:attribute name="type">submit</xsl:attribute>
<xsl:element name="i">
<xsl:attribute name="class">fa fa-search</xsl:attribute>
</xsl:element>
</xsl:element>
</xsl:element>
<xsl:element name="input">
<xsl:attribute name="placeholder">
<xsl:call-template name="fsfe-gettext"><xsl:with-param name="id" select="'search/placeholder'" /></xsl:call-template>
</xsl:attribute>
<xsl:attribute name="type">text</xsl:attribute>
<xsl:attribute name="name">q</xsl:attribute>
<xsl:attribute name="size">10</xsl:attribute>
<xsl:attribute name="class">form-control</xsl:attribute>
</xsl:element>
</xsl:element>
</xsl:element>
</xsl:element>
</xsl:element>
</xsl:element>
</xsl:if>

View File

@ -89,12 +89,12 @@
<text id="podcast">Podcast</text>
<text id="go-to">Go to:</text>
<text id="search">Search</text>
<text id="subscribe">Subscribe</text>
<text id="email">email address</text>
<text id="news">News</text>
<text id="breadcrumb-news">News</text>
<text id="pages">Pages</text>
<text id="receive-newsletter">Subscribe to FSFE's monthly newsletter</text>
<text id="subscribe-newsletter">Subscribe to the newsletter</text>
@ -215,4 +215,10 @@
<text id="size/small">small</text>
<text id="background/white">White background</text>
<text id="background/transparent">Transparent background</text>
<!-- Search function -->
<text id="search">Search</text>
<text id="search/placeholder">Search terms...</text>
<text id="search/notfound">No search results found. Please rephrase your query.</text>
<text id="search/empty">Your search query is empty.</text>
</data>

View File

@ -0,0 +1,21 @@
#menu-search-box {
display: inline-block;
vertical-align: middle;
form > div > input {
display: table-cell;
border-radius: 4px;
}
.btn-primary {
.btn;
background-color: transparent !important;
border: none !important;
color: @btn-primary-bg !important;
&:hover {
background-color: #2a7dae !important;
color: #fff !important;
border-color: #216280 !important;
}
}
}

View File

@ -2,6 +2,7 @@
@import "elements/figure";
@import "elements/banners";
@import "elements/podcast";
@import "elements/search-box";
@import "elements/sharebuttons";
@import "elements/people";
@import "pages/frontpage";

6
scripts/lunr-2.3.9.min.js vendored Normal file

File diff suppressed because one or more lines are too long

169
search/search.en.xhtml Normal file
View File

@ -0,0 +1,169 @@
<?xml version="1.0" encoding="UTF-8"?>
<html>
<version>1</version>
<head>
<title>Search</title>
<script type="text/javascript" src="/scripts/lunr-2.3.9.min.js"></script>
<script src="index.js"></script>
</head>
<body class="toplevel">
<h1>Search</h1>
<div id="introduction">
<p>
Find news articles and pages about topics your are interested in.
You can use one or multiple terms.
</p>
</div>
<p>
The search crawls through all site titles, teasers and tags, but
not the full article text. You will see maximum 15 results, sorted
in news and pages. The case of your term does not matter. If you do
not find what you were looking for, please try a variation of the
terms, or different words, and use the <a href="#tips">advanced
search features</a>.
</p>
<noscript>
<p>
JavaScript needs to be activated for the search functionality to
work. Usually, we do our best to avoid depending on this. If you do
not want to activate JavaScript, you can use an external search
engine which does not require it (for example <a
href="https://html.duckduckgo.com/html/">DuckDuckGo</a>), and use
the "site:fsfe.org" modifier in the query.
</p>
</noscript>
<form class="form-inline" method="GET" action="">
<div class="form-group">
<input type="text" class="form-control" id="search" name="q" aria-label="Search term" />
</div>
<button type="submit" class="btn btn-primary">Search</button>
</form>
<h2>Search results</h2>
<div id="search_results"></div>
<h2 id="tips">Tips for advanced searches</h2>
<p>
You can customise your searches to narrow down the results. Here
are a few examples, you can find more in the <a
href="https://lunrjs.com/guides/searching.html">documentation of
the library</a> we use.
</p>
<ul>
<li>
Wildcards: <code>communi*</code> will display results for e.g.
<em>community</em> and <em>communication</em>.
</li>
<li>
Presence: with <code>+router -patents consultation</code> you
define that <em>router</em> must be found in the results. All
results containing <em>patents</em> will be ruled out. The
presence of <em>consultation</em> is optional.
</li>
<li>
Fields: you can limit your search to the site titles with
<code>title:router</code>. Other fields are <code>teaser</code>,
<code>type</code> and <code>tags</code>.
</li>
<li>
Only news/pages: <code>+standard +type:page</code> only shows
pages with the word <em>standard</em>. The opposite is the
<em>news</em> type. Note the + signs to enforce both terms to be
present.
</li>
<li>
Boosts: you can increase weight of certain terms. With
<code>router^10 freedom</code> the weight of the first term is
10x higher.
</li>
<li>
Fuzzy matches: With <code>organisation~1</code>, you will find
results with <em>organisation</em> and <em>organization</em>. One
character in the findings can be different from your search term.
</li>
</ul>
<script>
const searchString = new URLSearchParams(window.location.search).get('q');
const locals = [document.documentElement.getAttribute("lang")];
if (!locals.includes('en')) {
locals.push('en');
}
const $target = document.querySelector('#search_results');
if (searchString) {
// Populate the field with any existing search string
document.querySelector('#search').value = searchString;
// Our index uses title as a key of the hashmap
const pagesByURL = pages.reduce((acc, curr) => {
acc[curr.url] = curr;
return acc;
}, {});
index = lunr(function() {
this.pipeline.remove(lunr.stopWordFilter);
this.pipeline.remove(lunr.trimmer);
this.field("title", { boost: 10 });
this.field("tags", { boost: 5 });
this.field("teaser");
this.field("type");
this.ref("url");
pages.forEach(function (page) {
this.add(page)
}, this)
});
// Do the search and filter out results not from the current local or English
let matches = index.search(searchString).filter(p => locals.some(local => p.ref.includes(local + ".html")));
function display_result(matches) {
// workaround xsl XML tag parsing madness
return '&lt;ul&gt;' + matches.map(p => {
title = pagesByURL[p.ref].title;
date = pagesByURL[p.ref].date;
if (date) {
return '<li>' + '<a href='&apos;+p.ref+&apos;'>'+title+'</a>'+' (' + date + ')</li>';
} else {
return '<li><a href='&apos;+p.ref+&apos;'>' + title + ' </a></li>';
}
}).join('') + '&lt;/ul&gt;';
}
if (matches.length > 0) {
matches = matches.slice(0, 15);
let [news, pages] = matches.reduce(([true_arr, false_arr], m)=> {
if (m.ref.includes('news') === false)
// return true_arr and append m to false_arr
return [true_arr, [...false_arr, m]]
else
return [[...true_arr,m], false_arr]
}, [[],[]]);
if (news.length > 0) {
news = news.sort((a, b) => pagesByURL[a.ref].date &lt; pagesByURL[b.ref].date);
$target.innerHTML = '<h3><translation id="news" /></h3>' + display_result(news);
}
if (pages.length > 0) {
$target.innerHTML += '<h3><translation id="pages" /></h3>' + display_result(pages);
}
} else {
$target.innerHTML = '<p><translation id="search/notfound" /></p>';
}
} else {
$target.innerHTML = '<p><translation id="search/empty" /></p>';
}
</script>
</body>
</html>

51
search/search.xsl Normal file
View File

@ -0,0 +1,51 @@
<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:import href="../fsfe.xsl" />
<!-- Create the search form. Doing this this way to add translations for placeholders and such -->
<xsl:template match="search-form">
<xsl:element name="form">
<xsl:attribute name="class">form-inline</xsl:attribute>
<xsl:attribute name="method">GET</xsl:attribute>
<xsl:attribute name="action"></xsl:attribute>
<xsl:element name="div">
<xsl:attribute name="class">form-group</xsl:attribute>
<xsl:element name="input">
<xsl:attribute name="type">text</xsl:attribute>
<xsl:attribute name="class">form-control</xsl:attribute>
<xsl:attribute name="id">search</xsl:attribute>
<xsl:attribute name="name">q</xsl:attribute>
<xsl:attribute name="aria-label">
<xsl:call-template name="fsfe-gettext">
<xsl:with-param name="id" select="'search/placeholder'" />
</xsl:call-template>
</xsl:attribute>
<xsl:attribute name="placeholder">
<xsl:call-template name="fsfe-gettext">
<xsl:with-param name="id" select="'search/placeholder'" />
</xsl:call-template>
</xsl:attribute>
</xsl:element> <!-- /input -->
</xsl:element> <!-- /div -->
<xsl:element name="button">
<xsl:attribute name="type">submit</xsl:attribute>
<xsl:attribute name="class">btn btn-primary</xsl:attribute>
<xsl:call-template name="fsfe-gettext">
<xsl:with-param name="id" select="'search'" />
</xsl:call-template>
</xsl:element> <!-- /button -->
</xsl:element> <!-- /form -->
</xsl:template>
<!-- Run fsfe-gettext for a given id, can be used directly from the XML file -->
<xsl:template match="translation">
<xsl:variable name="id"><xsl:value-of select="@id"/></xsl:variable>
<xsl:call-template name="fsfe-gettext">
<xsl:with-param name="id" select="$id" />
</xsl:call-template>
</xsl:template>
</xsl:stylesheet>

94
tools/index-website.py Normal file

File diff suppressed because one or more lines are too long