Source files of fsfe.org, pdfreaders.org, freeyourandroid.org, ilovefs.org, drm.info, and test.fsfe.org. Contribute: https://fsfe.org/contribute/web/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

build-test.pl.outdated-translations-patch 28KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953
  1. #! /usr/bin/perl
  2. #
  3. # build.pl - a tool for building FSF Europe web pages
  4. #
  5. # Copyright (C) 2003 Jonas Öberg
  6. #
  7. # This program is free software; you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation; either version 2 of the License, or
  10. # (at your option) any later version.
  11. #
  12. # This program is distributed in the hope that it will be useful, but
  13. # WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15. # General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU General Public License
  18. # along with this program; if not, write to the Free Software
  19. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20. # 02110-1301, USA.
  21. #
  22. use File::Find::Rule;
  23. use Getopt::Std;
  24. use File::Path;
  25. use File::Basename;
  26. use XML::LibXSLT;
  27. use XML::LibXML;
  28. use File::Copy;
  29. use POSIX qw(strftime);
  30. use IO::Handle;
  31. use IO::Select;
  32. use Socket;
  33. use Fcntl ':flock';
  34. use FindBin;
  35. use lib "$FindBin::Bin";
  36. require "comptree.pl";
  37. # This defines the focuses and their respective preferred / original
  38. # language. For example, it says that we should have a focus called
  39. # "se" (Sweden) which has the preferred language "sv" (Swedish).
  40. #
  41. # This also says that documents in the directory /se should be considered
  42. # as having the Swedish version as the original, and so on.
  43. #
  44. our %countries = (global => 'en');
  45. #our %countries = (
  46. # global => 'en',
  47. # de => 'de',
  48. # es => 'es',
  49. # it => 'it',
  50. # fr => 'fr',
  51. # se => 'sv' );
  52. #
  53. # This is a hash of all the languages that we have translations into, and their
  54. # respective names in the local language. Make sure that one entry exists
  55. # here for every language, or it won't be rendered.
  56. #
  57. # NOTE: Make sure that the language also is added to Apache configuration so
  58. # content negotiation works.
  59. #
  60. our %languages = (
  61. ar => 'العربيّة',
  62. bg => 'Български',
  63. ca => 'Català',
  64. cs => 'Česky',
  65. da => 'Dansk',
  66. de => 'Deutsch',
  67. el => 'Ελληνικά',
  68. en => 'English',
  69. es => 'Español',
  70. et => 'Eesti',
  71. fi => 'Suomi',
  72. fr => 'Français',
  73. hr => 'Hrvatski',
  74. hu => 'Magyar',
  75. it => 'Italiano',
  76. ku => 'Kurdî',
  77. mk => 'Mакедонски',
  78. nb => 'Norsk (bokmål)',
  79. nl => 'Nederlands',
  80. nn => 'Norsk (nynorsk)',
  81. pl => 'Polski',
  82. pt => 'Português',
  83. ro => 'Română',
  84. ru => 'Русский',
  85. sk => 'Slovenčina',
  86. sl => 'Slovenščina',
  87. sq => 'Shqip',
  88. sr => 'Srpski',
  89. sv => 'Svenska',
  90. tr => 'Türkçe',
  91. );
  92. our $current_date = strftime "%Y-%m-%d", localtime;
  93. our $current_time = strftime "%Y-%m-%d %H:%M:%S", localtime;
  94. # This static array contains files that can't be out of date
  95. our %cant_be_outdated = (
  96. "news/news" => 1
  97. );
  98. #
  99. # Parse the command line options. We need two; where to put the finished
  100. # pages and what to use as base for the input.
  101. #
  102. getopts('o:i:t:duqn', \%opts);
  103. unless ($opts{o}) {
  104. print STDERR "Usage: $0 [-q] [-u] [-d] [-n] [-t #] -o <output directory>\n";
  105. print STDERR " -q Quiet\n";
  106. print STDERR " -u Update only\n";
  107. print STDERR " -d Print some debug information\n";
  108. print STDERR " -n Don't write any files\n";
  109. print STDERR " -t Number of worker childs to create (default: 1)\n";
  110. exit 1;
  111. }
  112. # It might be nice to be able to specify this, but it will break things as
  113. # they are now. This is on the TODO list :-)
  114. $opts{i} = ".";
  115. $| = 1;
  116. $SIG{CHLD} = 'IGNORE';
  117. # Create XML and XSLT parser contexts. Also create the root note for the
  118. # above mentioned XML file (used to feed the XSL transformation).
  119. my $parser = XML::LibXML->new();
  120. my $xslt_parser = XML::LibXSLT->new();
  121. # Parse the global stylesheet
  122. my $global_style_doc = $parser->parse_file($opts{i}."/fsfe.xsl");
  123. my $global_stylesheet = $xslt_parser->parse_stylesheet($global_style_doc);
  124. #
  125. # First topic of today: create all directories we need. Instead of creating
  126. # these as they are used, we create them in a batch at the beginning of each
  127. # run, so we won't have to worry about them later.
  128. # Note though that this also REMOVES the previous paths. You don't want to
  129. # build directly into the production web tree.
  130. #
  131. my @dirs = File::Find::Rule->directory()
  132. ->in($opts{i});
  133. while (my ($path, undef) = each %countries) {
  134. print STDERR "Reseting path for $path\n" unless $opts{q};
  135. rmtree($opts{o}.'/'.$path) unless ($opts{u} || $opts{n});
  136. my @paths = map { $opts{o}."/$path/".$_ } grep(!/^\.\.?$/, @dirs);
  137. foreach (@paths) {
  138. print "Creating $_\n" if $opts{d};
  139. mkpath($_) unless $opts{n};
  140. }
  141. }
  142. #
  143. # Here starts our real work. First we get ourselves a list of all files
  144. # that we need to worry about and then single out the XHTML files. We
  145. # create a hash of hashes, %bases, which contains the basename of each
  146. # file, together with the translations that it exists in.
  147. #
  148. my @files = File::Find::Rule->file()
  149. ->in($opts{i});
  150. my %bases;
  151. foreach (grep(/\.xhtml$/, @files)) {
  152. $_ =~ s/^$opts{i}\/?// unless $opts{i} eq ".";
  153. my ($lang) = ($_ =~ /\.([a-z][a-z])\.xhtml$/);
  154. unless ($lang) {
  155. $lang = "en";
  156. }
  157. $_ =~ s/\.[a-z][a-z]\.xhtml$//;
  158. $_ =~ s/\.xhtml$//;
  159. $bases{$_}{$lang} = 1;
  160. }
  161. # Open the file where we will log all outdated and missing translations
  162. open (TRANSLATIONS, '>', "$opts{o}/translations.log");
  163. #
  164. # For each file, translation and focus, we create a new XML file. This will
  165. # contain all information that the XSL needs to produce a finished page.
  166. # The XML file will look like this:
  167. #
  168. # <buildinfo>
  169. # <trlist> <!-- All translations that this page exists in -->
  170. # <tr id="sv">Svenska</tr>
  171. # ...
  172. # </trlist>
  173. # <localmenuset> <!-- Local menu items for some directories -->
  174. # ...
  175. # </localmenuset>
  176. # <menuset> <!-- The menu items for the left hand bar -->
  177. # ...
  178. # </menuset>
  179. # <textset> <!-- The static text set for this language -->
  180. # ...
  181. # </textset>
  182. # <textsetbackup> <!-- The English textset as backup for missing translations -->
  183. # ...
  184. # </textsetbackup>
  185. # <document> <!-- The actual document, as read from the XHTML -->
  186. # <head>
  187. # <title>...</title>
  188. # <body>...</body>
  189. # </head>
  190. # </document>
  191. # </buildinfo>
  192. #
  193. # In addition to this, the buildinfo and document root will be equipped with
  194. # the following attributes:
  195. #
  196. # buildinfo/@original The language code of the original document
  197. # buildinfo/@filename The filename without language or trailing .html
  198. # buildinfo/@dirname The path to the file
  199. # buildinfo/@language The language that we're building into
  200. # buildinfo/@outdated Set to "yes" if the original is newer than this page
  201. # document/@language The language that this documents is in
  202. #
  203. #
  204. # $threads is the number of child processes to fork off to build the tree
  205. #
  206. unless ($threads = $opts{t}) {
  207. $threads = 1;
  208. }
  209. #
  210. # Start the required number of children, for each child we create a socket
  211. # pair to communicate between parent and child. This information is kept in
  212. # the %procs hash, which contains file handles for both child and parent.
  213. #
  214. foreach my $i (1..$threads) {
  215. $procs[$i]{child} = new IO::Handle;
  216. $procs[$i]{parent} = new IO::Handle;
  217. socketpair($procs[$i]{child}, $procs[$i]{parent}, AF_UNIX,
  218. SOCK_STREAM, PF_UNSPEC);
  219. $procs[$i]{child}->autoflush(1);
  220. $procs[$i]{parent}->autoflush(1);
  221. #$procs[$i]{child}->blocking(false);
  222. #$procs[$i]{parent}->blocking(false);
  223. if (fork()) {
  224. #
  225. # The parent doesn't do anything at this stage, except close one of
  226. # the filehandes not used.
  227. #
  228. close($procs[$i]{parent});
  229. } else {
  230. #
  231. # This is the main worker for the children, which wait for a command
  232. # to execute, either DIE or PROCESS. In the case of the first, the child
  233. # exists gracefully, in the case of the second, it calls on process()
  234. # to build the required page and languages.
  235. #
  236. # When waiting for the next page to be sent to it, the child sends NEXT
  237. # to the parent to signify that it's ready for the next command.
  238. #
  239. close($procs[$i]{child});
  240. my $io = $procs[$i]{parent};
  241. print $io "NEXT\n";
  242. while (!$io->error) {
  243. my $cmd = <$io>;
  244. if ($cmd =~ /DIE/) {
  245. exit;
  246. } elsif ($cmd =~ /PROCESS/) {
  247. chomp($cmd);
  248. my (undef, $file, $langs) = split(/\|/, $cmd);
  249. process($file, $langs);
  250. print $io "NEXT\n";
  251. }
  252. }
  253. exit;
  254. }
  255. }
  256. #
  257. # This sets up an IO::Select object with the filehandles of all children.
  258. # The parent uses this when looking for the next available child and blocks
  259. # until any child is ready.
  260. #
  261. my $s = IO::Select->new();
  262. foreach my $i (1..$threads) {
  263. $s->add($procs[$i]{child});
  264. }
  265. while (my ($file, $langs) = each %bases) {
  266. $s->can_read();
  267. my $done = 0;
  268. while (!$done) {
  269. foreach my $fh ($s->can_read()) {
  270. $cmd = <$fh>;
  271. if ($cmd =~ /NEXT/) {
  272. printf $fh "PROCESS|%s|%s\n", $file, join(':', keys(%{$langs}));
  273. $done = 1;
  274. last;
  275. }
  276. }
  277. }
  278. }
  279. #
  280. # When done, we send the DIE command to each child.
  281. #
  282. foreach my $i (1..$threads) {
  283. my $io = $procs[$i]{child};
  284. print $io "DIE\n";
  285. }
  286. #
  287. # This ensures a timely wait for every child to finish processing and shutdown.
  288. #
  289. while (wait() != -1) {
  290. sleep 2;
  291. }
  292. sub process {
  293. my ($file, $langs) = @_;
  294. #print "$file\n";
  295. print STDERR "Building $file.. \n" unless $opts{q};
  296. # Create the root note for the above mentioned XML file (used to feed the XSL
  297. # transformation).
  298. my $dom = XML::LibXML::Document->new("1.0", "iso-8859-1");
  299. my $root = $dom->createElement("buildinfo");
  300. $dom->setDocumentElement($root);
  301. #
  302. # Set the current date, to use for comparision in the XSLT.
  303. #
  304. $root->setAttribute("date", $current_date);
  305. #
  306. # Find original language. It's en, unless we're in the country specific
  307. # se/, fr/, de/ and so on, directories.
  308. #
  309. $root->setAttribute("original", "en");
  310. my $srcfocus = "global";
  311. if ($file =~ /^([a-z][a-z])\//) {
  312. $srcfocus = "$1";
  313. $root->setAttribute("original", $countries{$1});
  314. }
  315. $root->setAttribute("filename", "/$file");
  316. #
  317. # Set the directory name attribute
  318. #
  319. my (undef, $current_dir, undef) = fileparse($file);
  320. $root->setAttribute("dirname", "$current_dir");
  321. #
  322. # Find all translations for this document, and create the trlist set
  323. # for them.
  324. #
  325. my $trlist = $dom->createElement("trlist");
  326. foreach my $lang (split(/:/, $langs)) {
  327. my $tr = $dom->createElement("tr");
  328. $tr->setAttribute("id", $lang);
  329. $tr->appendText($languages{$lang});
  330. $trlist->appendChild($tr);
  331. }
  332. $root->appendChild($trlist);
  333. #
  334. # Load the file with local menu's
  335. #
  336. my $localmenu = "$opts{i}/localmenuinfo.xml";
  337. if (-f $localmenu) {
  338. my $menudoc = $dom->createElement("localmenuset");
  339. $root->appendChild($menudoc);
  340. clone_document($menudoc, $localmenu);
  341. }
  342. #
  343. # Load English backup texts
  344. #
  345. my $backup = $dom->createElement("textsetbackup");
  346. $root->appendChild($backup);
  347. clone_document($backup, $opts{i}."/tools/texts-en.xml");
  348. #
  349. # Transform it, once for every focus!
  350. #
  351. while (my ($dir, undef) = each %countries) {
  352. # If we handle a focus specific file, only process it in that focus
  353. next if (("$srcfocus" ne "global") && ("$dir" ne "$srcfocus"));
  354. print STDERR "$dir " unless $opts{q};
  355. #
  356. # And once for every language!
  357. #
  358. while (my ($lang, undef) = each %languages) {
  359. $root->setAttribute("language", $lang);
  360. #
  361. # This finds the source file to use. If we can't find a translation
  362. # into the language, it uses the english version instead, or that in
  363. # the local language. Or the first version it finds. This should be
  364. # made prettier.
  365. #
  366. my $document = $dom->createElement("document");
  367. $document->setAttribute("language", $lang);
  368. $root->appendChild($document);
  369. my $source = "$opts{i}/$file.$lang.xhtml";
  370. unless (-f $source) {
  371. my $missingsource = $source;
  372. if (-f "$opts{i}/$file.en.xhtml") {
  373. $document->setAttribute("language", "en");
  374. $source = "$opts{i}/$file.en.xhtml";
  375. } elsif (-f "$opts{i}/$file.".$root->getAttribute("original").".xhtml") {
  376. $document->setAttribute("language", $root->getAttribute("original"));
  377. $source = "$opts{i}/$file.".$root->getAttribute("original").".xhtml";
  378. } else {
  379. my $l = (keys %{$bases{$file}})[0];
  380. $document->setAttribute("language", $l);
  381. $source = "$opts{i}/$file.$l.xhtml";
  382. }
  383. if ($dir eq "global") {
  384. lock(*TRANSLATIONS);
  385. print TRANSLATIONS "$lang $missingsource $source\n";
  386. unlock(*TRANSLATIONS);
  387. }
  388. }
  389. if ( (stat("$opts{o}/$dir/$file.$lang.html"))[9] >
  390. (stat($source))[9] && $opts{u} && ! -f "$opts{i}/$file.xsl" ) {
  391. next;
  392. }
  393. #
  394. # If there is a stylesheet (XSL) in any directory with the same
  395. # name as the directory (e.g. the directory is called "foo" and
  396. # there is a file in that directory called "foo/foo.xsl", this
  397. # stylesheet will replace the global stylesheet "/fsfe.xsl".
  398. #
  399. # my $subsite_stylesheet;
  400. # my $subsite_style_doc_file = dirname("$opts{i}/$file.$lang.xhtml")."/".basename(dirname("$opts{i}/$file.$lang.xhtml")).".xsl";
  401. # if (-f $subsite_style_doc_file && ! -f "$opts{i}/$file.xsl") {
  402. # my $subsite_style_doc = $parser->parse_file($subsite_style_doc_file);
  403. # $subsite_stylesheet = $xslt_parser->parse_stylesheet($subsite_style_doc);
  404. # }
  405. #
  406. # Here begins automated magic for those pages which we need to
  407. # assemble other sets of informations for first (automatically
  408. # updated pages).
  409. #
  410. if (-f "$opts{i}/$file.xsl") {
  411. #
  412. # Settle down please, children. First we remove all previous
  413. # document leftovers.
  414. #
  415. foreach ($root->getElementsByTagName("document")) {
  416. $root->removeChild($_);
  417. }
  418. $root->appendChild($document);
  419. # Create the <timestamp> tag automatically for these documents
  420. my $timestamp = $dom->createElement("timestamp");
  421. $timestamp->appendText("\$"."Date: ".$current_time." \$ \$"."Author: automatic \$");
  422. $document->appendChild($timestamp);
  423. #
  424. # Get the list of sources and create the files hash. The files
  425. # hash contains the base name for each file we want to use, and
  426. # then the language for it as a value. It prefers a file in the
  427. # language we're building into, but will accept an English file as
  428. # a substitute.
  429. #
  430. # "Learn all that is learnable and return that information
  431. # to the Creator."
  432. #
  433. open(IN, '<', "$opts{i}/$file.sources");
  434. my @auto_sources = <IN>;
  435. close IN;
  436. my %files;
  437. foreach (@auto_sources) {
  438. if (/(.*):[a-z,]*global/ || /(.*):[a-z,]*$dir/) {
  439. foreach my $f (glob("$1*")) {
  440. if ($f =~ /(.*)\.([a-z][a-z])\.xml$/) {
  441. if (!$files{$1}) {
  442. $files{$1} = $2;
  443. } elsif ($2 eq $lang) {
  444. $files{$1} = $2;
  445. } elsif (($2 eq "en") &&
  446. ($files{$1} ne $lang)) {
  447. $files{$1} = $2;
  448. }
  449. }
  450. }
  451. }
  452. }
  453. #
  454. # With that information, we load the source document and create
  455. # a new element in it, called <set>, which will hold the combined
  456. # knowledge of all the sets in the source files.
  457. #
  458. my $sourcedoc = $parser->parse_file($source);
  459. $sourcedoc->documentElement->setAttribute("date", $current_date);
  460. $sourcedoc->documentElement->setAttribute("lang", $lang);
  461. my $auto_data = $sourcedoc->createElement("set");
  462. while (my ($base, $l) = each %files) {
  463. if (($dir eq "global") && ($l ne $lang)) {
  464. lock(*TRANSLATIONS);
  465. print TRANSLATIONS "$lang $base.$lang.xml $base.$l.xml\n";
  466. unlock(*TRANSLATIONS);
  467. }
  468. print STDERR "Loading $base.$l.xml\n" if $opts{d};
  469. my $source_data = $parser->parse_file("$base.$l.xml");
  470. foreach ($source_data->documentElement->childNodes) {
  471. my $c = $_->cloneNode(1);
  472. # add the filename to nodes (news, events, …) so that we can use it as an identifier (e.g. for RSS)
  473. if (ref($c) eq "XML::LibXML::Element") {
  474. $base =~ /.*[\/_]([^\/_]*$)/;
  475. $c->setAttribute( "filename", $1 );
  476. }
  477. $auto_data->appendChild($c);
  478. }
  479. }
  480. $sourcedoc->documentElement->appendChild($auto_data);
  481. #
  482. # Get the appropriate textset for this language. If one can't be
  483. # found, use the English. (I hope this never happens)
  484. #
  485. my $textlang = $lang;
  486. unless (-f $opts{i}."/tools/texts-content-$textlang.xml") {
  487. $textlang = "en";
  488. }
  489. my $textdoc = $sourcedoc->createElement("textset-content");
  490. $auto_data->appendChild($textdoc);
  491. clone_document($textdoc, $opts{i}."/tools/texts-content-$textlang.xml");
  492. # Get also backup texts from the English file
  493. my $textdocbak = $sourcedoc->createElement("textset-content-backup");
  494. $auto_data->appendChild($textdocbak);
  495. clone_document($textdocbak, $opts{i}."/tools/texts-content-en.xml");
  496. # TODO: optimise getting texts-content-xx.xml and texts-content-en.xml,
  497. # since it does not depend on the xsl file being treated, we should do it only once!
  498. #
  499. # Transform the document using the XSL file and then push the
  500. # result into the <document> element of the document we're building.
  501. #
  502. my $style_doc = $parser->parse_file("$opts{i}/$file.xsl");
  503. my $stylesheet = $xslt_parser->parse_stylesheet($style_doc);
  504. my $results = $stylesheet->transform($sourcedoc);
  505. foreach ($results->documentElement->childNodes) {
  506. my $c = $_->cloneNode(1);
  507. $document->appendChild($c);
  508. }
  509. #
  510. # Now, while we're just at it, we create the RSS feeds if we want any
  511. #
  512. if (-f "$opts{i}/$file.rss.xsl") {
  513. my $style_doc = $parser->parse_file("$opts{i}/$file.rss.xsl");
  514. my $stylesheet = $xslt_parser->parse_stylesheet($style_doc);
  515. my $results = $stylesheet->transform($sourcedoc);
  516. $stylesheet->output_file($results, "$opts{o}/$dir/$file.$lang.rss")
  517. unless $opts{n};
  518. }
  519. #
  520. # and possibly the corresponding iCal (ics) file
  521. #
  522. if (-f "$opts{i}/$file.ics.xsl") {
  523. my $style_doc = $parser->parse_file("$opts{i}/$file.ics.xsl");
  524. my $stylesheet = $xslt_parser->parse_stylesheet($style_doc);
  525. my $results = $stylesheet->transform($sourcedoc);
  526. $stylesheet->output_file($results, "$opts{o}/$dir/$file.$lang.ics")
  527. unless $opts{n};
  528. }
  529. } else {
  530. #
  531. # If this wasn't an automatically updating document, we simply
  532. # clone the contents of the source file into the document.
  533. #
  534. clone_document($document, $source);
  535. }
  536. #
  537. # Find out if this translation is to be regarded as outdated or not.
  538. # A translation is deemed outdated if it is more than 2 hours older
  539. # than the original. This makes sure a translation committed together
  540. # with the original (but maybe a second earlier) isn't marked outdated.
  541. #
  542. my $originalsource = "$file.".$root->getAttribute("original").".xhtml";
  543. my $comment;
  544. my $old_outdated = 0;
  545. if (( stat("$opts{i}/$originalsource"))[9] > (stat($source))[9] + 7200
  546. and not $cant_be_outdated{$file} ) {
  547. $old_outdated = 1;
  548. } else {
  549. }
  550. my $new_outdated = 0;
  551. if ( not -e "$opts{i}/$originalsource" ) {
  552. # TODO: do something
  553. } else {
  554. my ($equal, $err) = areEqual( "$opts{i}/$originalsource", $source );
  555. if ( not $equal and not $cant_be_outdated{$file} ) {
  556. #print "$err\n";
  557. $root->setAttribute("outdated", "yes");
  558. $new_outdated = 1;
  559. # register information about the outdated status
  560. my $info = $dom->createElement("outdated-info");
  561. my $code = $dom->createElement("code");
  562. $code->appendChild($dom->createElement("br"));
  563. foreach $line (split(/\n/, $err)) {
  564. $code->appendText($line);
  565. $code->appendChild($dom->createElement("br"));
  566. }
  567. $info->appendChild($code);
  568. $document->appendChild($info);
  569. } else {
  570. $root->setAttribute("outdated", "no");
  571. }
  572. }
  573. if ($dir eq "global" and $new_outdated != $old_outdated ) {
  574. if ( $new_outdated == 0 and $old_outdated == 1 ) {
  575. $comment = "timestamp-OUT,structure-OK";
  576. } elsif ( $new_outdated == 1 and $old_outdated == 0 ) {
  577. $comment = "timestamp-OK,structure-OUT";
  578. } else {
  579. $comment = "-";
  580. }
  581. lock(*TRANSLATIONS);
  582. print TRANSLATIONS "$lang $source $originalsource $comment\n";
  583. unlock(*TRANSLATIONS);
  584. }
  585. #
  586. # Get the appropriate textset for this language. If one can't be
  587. # found, use the English. (I hope this never happens)
  588. #
  589. my $textlang = $lang;
  590. unless (-f $opts{i}."/tools/texts-$textlang.xml") {
  591. $textlang = "en";
  592. }
  593. my $textdoc = $dom->createElement("textset");
  594. $root->appendChild($textdoc);
  595. clone_document($textdoc, $opts{i}."/tools/texts-$textlang.xml");
  596. #
  597. # Read the fundraising text, if it exists.
  598. #
  599. if (-f $opts{i}."/fundraising.$lang.xml") {
  600. my $fundraisingdoc = $dom->createElement("fundraising");
  601. $root->appendChild($fundraisingdoc);
  602. clone_document($fundraisingdoc, $opts{i}."/fundraising.$lang.xml");
  603. } elsif (-f $opts{i}."/fundraising.en.xml") {
  604. my $fundraisingdoc = $dom->createElement("fundraising");
  605. $root->appendChild($fundraisingdoc);
  606. clone_document($fundraisingdoc, $opts{i}."/fundraising.en.xml");
  607. }
  608. #
  609. # And then we do the same thing for the menues. But first we take the
  610. # global menu here, then we add any information that is specific to
  611. # the focus.
  612. #
  613. foreach ($root->getElementsByTagName("menuset")) {
  614. $root->removeChild($_);
  615. }
  616. my %menu;
  617. foreach ('global', $dir) {
  618. if (-f $opts{i}."/tools/menu-$_.xml") {
  619. my $menudoc = $parser->parse_file($opts{i}."/tools/menu-$_.xml");
  620. foreach my $n ($menudoc->documentElement->getElementsByTagName("menu")) {
  621. $menu{$n->getAttribute("id")} = $n;
  622. }
  623. }
  624. }
  625. my $menuroot = $dom->createElement("menuset");
  626. while (my ($id, $n) = each %menu) {
  627. my $m = $n->cloneNode(1);
  628. $menuroot->appendChild($m);
  629. }
  630. $root->appendChild($menuroot);
  631. # <start addendum> (TODO: transform this into a function)
  632. #
  633. # Get the list of sources and create the files hash. The files
  634. # hash contains the base name for each file we want to use, and
  635. # then the language for it as a value. It prefers a file in the
  636. # language we're building into, but will accept an English file as
  637. # a substitute.
  638. #
  639. # "Learn all that is learnable and return that information
  640. # to the Creator."
  641. #
  642. open(IN, '<', "$opts{i}/fsfe.sources");
  643. my @auto_sources = <IN>;
  644. close IN;
  645. my %files;
  646. foreach (@auto_sources) {
  647. if (/(.*):[a-z,]*global/ || /(.*):[a-z,]*$dir/) {
  648. foreach my $f (glob("$1*")) {
  649. if ($f =~ /(.*)\.([a-z][a-z])\.xml$/) {
  650. if (!$files{$1}) {
  651. $files{$1} = $2;
  652. } elsif ($2 eq $lang) {
  653. $files{$1} = $2;
  654. } elsif (($2 eq "en") &&
  655. ($files{$1} ne $lang)) {
  656. $files{$1} = $2;
  657. }
  658. }
  659. }
  660. }
  661. }
  662. #
  663. # With that information, we load the source document and create
  664. # a new element in it, called <set>, which will hold the combined
  665. # knowledge of all the sets in the source files.
  666. #
  667. foreach ($root->getElementsByTagName("set")) {
  668. $root->removeChild($_);
  669. }
  670. my $auto_data = $dom->createElement("set");
  671. while (my ($base, $l) = each %files) {
  672. if (($dir eq "global") && ($l ne $lang)) {
  673. lock(*TRANSLATIONS);
  674. print TRANSLATIONS "$lang $base.$lang.xml $base.$l.xml\n";
  675. unlock(*TRANSLATIONS);
  676. }
  677. print STDERR "Loading $base.$l.xml\n" if $opts{d};
  678. my $source_data = $parser->parse_file("$base.$l.xml");
  679. foreach ($source_data->documentElement->childNodes) {
  680. my $c = $_->cloneNode(1);
  681. # add the filename to nodes (news, events, …) so that we can use it as an identifier (e.g. for RSS)
  682. if (ref($c) eq "XML::LibXML::Element") {
  683. $base =~ /.*[\/_]([^\/_]*$)/;
  684. $c->setAttribute( "filename", $1 );
  685. }
  686. $auto_data->appendChild($c);
  687. }
  688. }
  689. $root->appendChild($auto_data);
  690. # <end addendum>
  691. if ( $lang eq "fr" and $file eq $file_to_treat ) {
  692. print "--->outputting test2.xml\n";
  693. open (TEST, '>', "/home/nicolas/FSFE/fsfe-web-out/test2.xml");
  694. print TEST $dom->toString();
  695. close (TEST);
  696. }
  697. #
  698. # Do the actual transformation. (through fsfe.xsl)
  699. #
  700. my $results = $global_stylesheet->transform($dom);
  701. #
  702. # In post-processing, we replace links pointing back to ourselves
  703. # so that they point to the correct language.
  704. #
  705. foreach ($results->documentElement->getElementsByTagName("a")) {
  706. my $href = $_->getAttribute("href");
  707. if ($href =~ /^http:\/\/test.fsfe.org/) {
  708. if ($_->textContent != "Our global work") {
  709. $href =~ s/http:\/\/test.fsfe.org//;
  710. }
  711. }
  712. if (($href !~ /^http/) && ($href !~ /^#/)) {
  713. # Save possible anchor and remove it from URL
  714. my $anchor = $href;
  715. if (!($anchor =~ s/.*#/#/)) {
  716. $anchor = "";
  717. }
  718. $href =~ s/#.*//;
  719. # proces URL
  720. if (($href =~ /\.html$/) && ($href !~ /\.[a-z][a-z]\.html$/)) {
  721. $href =~ s/\.html$/\.$lang.html/;
  722. } elsif (($href =~ /\.rss$/) && ($href !~ /\.[a-z][a-z]\.rss$/)) {
  723. $href =~ s/\.rss$/\.$lang.rss/;
  724. } elsif (($href =~ /\.ics$/) && ($href !~ /\.[a-z][a-z]\.ics$/)) {
  725. $href =~ s/\.ics$/\.$lang.ics/;
  726. } else {
  727. if (-d $opts{i}."/$href") {
  728. $href =~ s/\/?$/\/index.$lang.html/;
  729. } elsif ($href =~ /\/\w+$/) {
  730. $href .= ".$lang.html";
  731. }
  732. }
  733. # replace anchor
  734. $href .= $anchor;
  735. $_->setAttribute("href", $href);
  736. }
  737. }
  738. print "Writing: $opts{o}/$dir/$file.$lang.html\n" if $opts{d};
  739. unless ($opts{n}) {
  740. # if ($subsite_stylesheet) {
  741. # $subsite_stylesheet->output_file($results, "$opts{o}/$dir/$file.$lang.html");
  742. # } else {
  743. $global_stylesheet->output_file($results, "$opts{o}/$dir/$file.$lang.html");
  744. # }
  745. }
  746. # Add foo.html.xx link which is used by Apache's MultiViews option when
  747. # a user enters foo.html as URL.
  748. link("$opts{o}/$dir/$file.$lang.html", "$opts{o}/$dir/$file.html.$lang")
  749. unless $opts{n};
  750. }
  751. }
  752. print STDERR "\n" unless $opts{q};
  753. }
  754. # Close the logfile for outdated and missing translations
  755. close (TRANSLATIONS);
  756. print STDERR "Fixing index links\n" unless $opts{q};
  757. while (my ($path, undef) = each %countries) {
  758. my @dirs = File::Find::Rule->directory()
  759. ->in($opts{o}."/$path");
  760. foreach (@dirs) {
  761. my $base = basename($_);
  762. while (my ($lang, undef) = each %languages) {
  763. if (-f "$_/$base.$lang.html" &&
  764. ! -f "$_/index.$lang.html") {
  765. link("$_/$base.$lang.html", "$_/index.$lang.html")
  766. unless $opts{n};
  767. link("$_/$base.html.$lang", "$_/index.html.$lang")
  768. unless $opts{n};
  769. }
  770. }
  771. }
  772. }
  773. #
  774. # For all files that are not XHTML source files, we copy them verbatim to
  775. # the final location, for each focus. These should be links instead to
  776. # prevent us from wasting disk space.
  777. #
  778. print STDERR "Copying misc files\n" unless $opts{q};
  779. foreach (grep(!/\.sources$/, grep(!/\.xsl$/, grep(!/\.xml$/, grep(!/\.xhtml$/,
  780. @files))))) {
  781. while (my ($dir, undef) = each %countries) {
  782. if (-f "$opts{i}/$_" && !$opts{n}) {
  783. link("$opts{i}/$_", "$opts{o}/$dir/$_");
  784. }
  785. }
  786. }
  787. #
  788. # Helper function that clones a document. It accepts an XML node and
  789. # a filename as parameters. Using this, it loads the source file into
  790. # the XML node.
  791. #
  792. sub clone_document {
  793. my ($doc, $source) = @_;
  794. my $root = $doc->parentNode;
  795. print "Source: $source\n" if $opts{d};
  796. foreach ($root->getElementsByTagName($doc->nodeName)) {
  797. $root->removeChild($_);
  798. }
  799. $root->appendChild($doc);
  800. my $parser = XML::LibXML->new();
  801. $parser->load_ext_dtd(0);
  802. $parser->recover(1);
  803. my $sourcedoc = $parser->parse_file($source);
  804. foreach ($sourcedoc->documentElement->childNodes) {
  805. $_->unbindNode();
  806. my $n = $_->cloneNode(1);
  807. $doc->appendChild($n);
  808. }
  809. if ($sourcedoc->documentElement->getAttribute("external")) {
  810. $doc->setAttribute("external", "yes");
  811. }
  812. if ($sourcedoc->documentElement->getAttribute("newsdate")) {
  813. # necessary for xhtml news files
  814. $doc->setAttribute("newsdate", $sourcedoc->documentElement->getAttribute("newsdate"));
  815. }
  816. if ($sourcedoc->documentElement->getAttribute("type")) {
  817. # necessary to differentiate news and newsletter pages
  818. # TODO: find a way to copy all such attributes!
  819. $doc->setAttribute("type", $sourcedoc->documentElement->getAttribute("type"));
  820. }
  821. }
  822. #
  823. # Helper functions to lock and unlock the translation log.
  824. #
  825. sub lock {
  826. my ($fh) = @_;
  827. flock($fh, LOCK_EX);
  828. seek($fh, 0, 2);
  829. }
  830. sub unlock {
  831. my ($fh) = @_;
  832. flock($fh, LOCK_UN);
  833. }