Source files of fsfe.org, pdfreaders.org, freeyourandroid.org, ilovefs.org, drm.info, and test.fsfe.org. Contribute: https://fsfe.org/contribute/web/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

build-test.pl 30KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908
  1. #! /usr/bin/perl
  2. #
  3. # build.pl - a tool for building FSFE web pages
  4. #
  5. # Copyright (C) 2003 Jonas Öberg
  6. #
  7. # This program is free software; you can redistribute it and/or modify
  8. # it under the terms of the GNU General Public License as published by
  9. # the Free Software Foundation; either version 2 of the License, or
  10. # (at your option) any later version.
  11. #
  12. # This program is distributed in the hope that it will be useful, but
  13. # WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15. # General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU General Public License
  18. # along with this program; if not, write to the Free Software
  19. # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20. # 02110-1301, USA.
  21. #
  22. use File::Find::Rule;
  23. use Getopt::Std;
  24. use File::Path;
  25. use File::Basename;
  26. use XML::LibXSLT;
  27. use XML::LibXML;
  28. use File::Copy;
  29. use POSIX qw(strftime);
  30. use IO::Handle;
  31. use IO::Select;
  32. use Socket;
  33. use Fcntl ':flock';
  34. use FindBin;
  35. use lib "$FindBin::Bin";
  36. require "comptree.pl";
  37. # This defines the focuses and their respective preferred / original
  38. # language. For example, it says that we should have a focus called
  39. # "se" (Sweden) which has the preferred language "sv" (Swedish).
  40. #
  41. # This also says that documents in the directory /se should be considered
  42. # as having the Swedish version as the original, and so on.
  43. #
  44. our %countries = (global => 'en');
  45. #our %countries = (
  46. # global => 'en',
  47. # de => 'de',
  48. # es => 'es',
  49. # it => 'it',
  50. # fr => 'fr',
  51. # se => 'sv' );
  52. #
  53. # This is a hash of all the languages that we have translations into, and their
  54. # respective names in the local language. Make sure that one entry exists
  55. # here for every language, or it won't be rendered.
  56. #
  57. # NOTE: Make sure that the language also is added to Apache configuration so
  58. # content negotiation works.
  59. #
  60. our %languages = (
  61. ar => 'العربيّة',
  62. bg => 'Български',
  63. ca => 'Català',
  64. cs => 'Česky',
  65. da => 'Dansk',
  66. de => 'Deutsch',
  67. el => 'Ελληνικά',
  68. en => 'English',
  69. es => 'Español',
  70. et => 'Eesti',
  71. fi => 'Suomi',
  72. fr => 'Français',
  73. hr => 'Hrvatski',
  74. hu => 'Magyar',
  75. it => 'Italiano',
  76. ku => 'Kurdî',
  77. mk => 'Mакедонски',
  78. nb => 'Norsk (bokmål)',
  79. nl => 'Nederlands',
  80. nn => 'Norsk (nynorsk)',
  81. pl => 'Polski',
  82. pt => 'Português',
  83. ro => 'Română',
  84. ru => 'Русский',
  85. sk => 'Slovenčina',
  86. sl => 'Slovenščina',
  87. sq => 'Shqip',
  88. sr => 'Srpski',
  89. sv => 'Svenska',
  90. tr => 'Türkçe',
  91. );
  92. our $current_date = strftime "%Y-%m-%d", localtime;
  93. our $current_time = strftime "%Y-%m-%d %H:%M:%S", localtime;
  94. # This static array contains files that can't be out of date
  95. our %cant_be_outdated = (
  96. "news/news" => 1,
  97. "index" => 1
  98. );
  99. #
  100. # Parse the command line options. We need two; where to put the finished
  101. # pages and what to use as base for the input.
  102. #
  103. getopts('o:i:t:duqn', \%opts);
  104. unless ($opts{o}) {
  105. print STDERR "Usage: $0 [-q] [-u] [-d] [-n] [-t #] -o <output directory>\n";
  106. print STDERR " -q Quiet\n";
  107. print STDERR " -u Update only\n";
  108. print STDERR " -d Print some debug information\n";
  109. print STDERR " -n Don't write any files\n";
  110. print STDERR " -t Number of worker childs to create (default: 1)\n";
  111. exit 1;
  112. }
  113. # It might be nice to be able to specify this, but it will break things as
  114. # they are now. This is on the TODO list :-)
  115. $opts{i} = ".";
  116. $| = 1;
  117. $SIG{CHLD} = 'IGNORE';
  118. # Create XML and XSLT parser contexts. Also create the root note for the
  119. # above mentioned XML file (used to feed the XSL transformation).
  120. my $parser = XML::LibXML->new('encoding'=>'utf-8');
  121. my $xslt_parser = XML::LibXSLT->new('encoding'=>'utf-8');
  122. # Parse the global stylesheet
  123. my $global_style_doc = $parser->parse_file($opts{i}."/fsfe.xsl");
  124. my $global_stylesheet = $xslt_parser->parse_stylesheet($global_style_doc);
  125. #
  126. # First topic of today: create all directories we need. Instead of creating
  127. # these as they are used, we create them in a batch at the beginning of each
  128. # run, so we won't have to worry about them later.
  129. # Note though that this also REMOVES the previous paths. You don't want to
  130. # build directly into the production web tree.
  131. #
  132. my @dirs = File::Find::Rule->directory()
  133. ->in($opts{i});
  134. while (my ($path, undef) = each %countries) {
  135. print STDERR "Resetting path for $path\n" unless $opts{q};
  136. rmtree($opts{o}.'/'.$path) unless ($opts{u} || $opts{n});
  137. my @paths = map { $opts{o}."/$path/".$_ } grep(!/^\.\.?$/, @dirs);
  138. foreach (@paths) {
  139. print "Creating $_\n" if $opts{d};
  140. mkpath($_) unless $opts{n};
  141. }
  142. }
  143. #
  144. # Here starts our real work. First we get ourselves a list of all files
  145. # that we need to worry about and then single out the XHTML files. We
  146. # create a hash of hashes, %bases, which contains the basename of each
  147. # file, together with the translations that it exists in.
  148. #
  149. my @files = File::Find::Rule->file()
  150. ->in($opts{i});
  151. my %bases;
  152. foreach (grep(/\.xhtml$/, @files)) {
  153. $_ =~ s/^$opts{i}\/?// unless $opts{i} eq ".";
  154. my ($lang) = ($_ =~ /\.([a-z][a-z])\.xhtml$/);
  155. unless ($lang) { $lang = "en"; }
  156. $_ =~ s/\.[a-z][a-z]\.xhtml$//;
  157. $_ =~ s/\.xhtml$//;
  158. $bases{$_}{$lang} = 1;
  159. }
  160. # Open the file where we will log all outdated and missing translations
  161. open (TRANSLATIONS, '>', "$opts{o}/translations.log");
  162. #
  163. # For each file, translation and focus, we create a new XML file. This will
  164. # contain all information that the XSL needs to produce a finished page.
  165. # The XML file will look like this:
  166. #
  167. # <buildinfo>
  168. # <trlist> <!-- All translations that this page exists in -->
  169. # <tr id="sv">Svenska</tr>
  170. # ...
  171. # </trlist>
  172. # <localmenuset> <!-- Local menu items for some directories -->
  173. # ...
  174. # </localmenuset>
  175. # <menuset> <!-- The menu items for the left hand bar -->
  176. # ...
  177. # </menuset>
  178. # <textset> <!-- The static text set for this language -->
  179. # ...
  180. # </textset>
  181. # <textsetbackup> <!-- The English textset as backup for missing translations -->
  182. # ...
  183. # </textsetbackup>
  184. # <document> <!-- The actual document, as read from the XHTML -->
  185. # <head>
  186. # <title>...</title>
  187. # <body>...</body>
  188. # </head>
  189. # </document>
  190. # </buildinfo>
  191. #
  192. # In addition to this, the buildinfo and document root will be equipped with
  193. # the following attributes:
  194. #
  195. # buildinfo/@original The language code of the original document
  196. # buildinfo/@filename The filename without language or trailing .html
  197. # buildinfo/@dirname The path to the file
  198. # buildinfo/@language The language that we're building into
  199. # buildinfo/@outdated Set to "yes" if the original is newer than this page
  200. # document/@language The language that this documents is in
  201. #
  202. #
  203. # $threads is the number of child processes to fork off to build the tree
  204. #
  205. unless ($threads = $opts{t}) {
  206. $threads = 1;
  207. }
  208. #
  209. # Start the required number of children, for each child we create a socket
  210. # pair to communicate between parent and child. This information is kept in
  211. # the %procs hash, which contains file handles for both child and parent.
  212. #
  213. foreach my $i (1..$threads) {
  214. $procs[$i]{child} = new IO::Handle;
  215. $procs[$i]{parent} = new IO::Handle;
  216. socketpair($procs[$i]{child}, $procs[$i]{parent}, AF_UNIX,
  217. SOCK_STREAM, PF_UNSPEC);
  218. $procs[$i]{child}->autoflush(1);
  219. $procs[$i]{parent}->autoflush(1);
  220. #$procs[$i]{child}->blocking(false);
  221. #$procs[$i]{parent}->blocking(false);
  222. if (fork()) {
  223. #
  224. # The parent doesn't do anything at this stage, except close one of
  225. # the filehandes not used.
  226. #
  227. close($procs[$i]{parent});
  228. } else {
  229. #
  230. # This is the main worker for the children, which wait for a command
  231. # to execute, either DIE or PROCESS. In the case of the first, the child
  232. # exists gracefully, in the case of the second, it calls on process()
  233. # to build the required page and languages.
  234. #
  235. # When waiting for the next page to be sent to it, the child sends NEXT
  236. # to the parent to signify that it's ready for the next command.
  237. #
  238. close($procs[$i]{child});
  239. my $io = $procs[$i]{parent};
  240. print $io "NEXT\n";
  241. while (!$io->error) {
  242. my $cmd = <$io>;
  243. if ($cmd =~ /DIE/) {
  244. exit;
  245. } elsif ($cmd =~ /PROCESS/) {
  246. chomp($cmd);
  247. my (undef, $file, $langs) = split(/\|/, $cmd);
  248. process($file, $langs);
  249. print $io "NEXT\n";
  250. }
  251. }
  252. exit;
  253. }
  254. }
  255. #
  256. # This sets up an IO::Select object with the filehandles of all children.
  257. # The parent uses this when looking for the next available child and blocks
  258. # until any child is ready.
  259. #
  260. my $s = IO::Select->new();
  261. foreach my $i (1..$threads) {
  262. $s->add($procs[$i]{child});
  263. }
  264. while (my ($file, $langs) = each %bases) {
  265. $s->can_read();
  266. my $done = 0;
  267. while (!$done) {
  268. foreach my $fh ($s->can_read()) {
  269. $cmd = <$fh>;
  270. if ($cmd =~ /NEXT/) {
  271. printf $fh "PROCESS|%s|%s\n", $file, join(':', keys(%{$langs}));
  272. $done = 1;
  273. last;
  274. }
  275. }
  276. }
  277. }
  278. #
  279. # When done, we send the DIE command to each child.
  280. #
  281. foreach my $i (1..$threads) {
  282. my $io = $procs[$i]{child};
  283. print $io "DIE\n";
  284. }
  285. #
  286. # This ensures a timely wait for every child to finish processing and shutdown.
  287. #
  288. while (wait() != -1) {
  289. sleep 2;
  290. }
  291. sub process {
  292. my ($file, $langs) = @_;
  293. #print "$file\n";
  294. print STDERR "Building $file.. \n" unless $opts{q};
  295. # Create the root note for the above mentioned XML file (used to feed the XSL
  296. # transformation).
  297. my $dom = XML::LibXML::Document->new("1.0", "utf-8");
  298. my $root = $dom->createElement("buildinfo");
  299. $dom->setDocumentElement($root);
  300. #
  301. # Set the current date, to use for comparision in the XSLT.
  302. #
  303. $root->setAttribute("date", $current_date);
  304. #
  305. # Find original language. It's en, unless we're in the country specific
  306. # se/, fr/, de/ and so on, directories.
  307. #
  308. $root->setAttribute("original", "en");
  309. my $srcfocus = "global";
  310. if ($file =~ /^([a-z][a-z])\//) {
  311. $srcfocus = "$1";
  312. $root->setAttribute("original", $countries{$1});
  313. }
  314. $root->setAttribute("filename", "/$file");
  315. #
  316. # Set the directory name attribute
  317. #
  318. my (undef, $current_dir, undef) = fileparse($file);
  319. $root->setAttribute("dirname", "$current_dir");
  320. #
  321. # Find all translations for this document, and create the trlist set
  322. # for them.
  323. #
  324. my $trlist = $dom->createElement("trlist");
  325. foreach my $lang (split(/:/, $langs)) {
  326. my $tr = $dom->createElement("tr");
  327. $tr->setAttribute("id", $lang);
  328. $tr->appendText($languages{$lang});
  329. $trlist->appendChild($tr);
  330. }
  331. $root->appendChild($trlist);
  332. #
  333. # Load the file with local menu's
  334. #
  335. my $localmenu = "$opts{i}/localmenuinfo.xml";
  336. if (-f $localmenu) {
  337. my $menudoc = $dom->createElement("localmenuset");
  338. $root->appendChild($menudoc);
  339. clone_document($menudoc, $localmenu);
  340. }
  341. #
  342. # Load English backup texts
  343. #
  344. my $backup = $dom->createElement("textsetbackup");
  345. $root->appendChild($backup);
  346. clone_document($backup, $opts{i}."/tools/texts-en.xml");
  347. #
  348. # Transform it, once for every focus!
  349. #
  350. while (my ($dir, undef) = each %countries) {
  351. # If we handle a focus specific file, only process it in that focus
  352. # -> we don't handle focus-specific files anymore, commenting next line out, since it's causing errors
  353. #next if (("$srcfocus" ne "global") && ("$dir" ne "$srcfocus"));
  354. print STDERR "$dir " unless $opts{q};
  355. #
  356. # And once for every language!
  357. #
  358. while (my ($lang, undef) = each %languages) {
  359. $root->setAttribute("language", $lang);
  360. #
  361. # This finds the source file to use. If we can't find a translation
  362. # into the language, it uses the english version instead, or that in
  363. # the local language. Or the first version it finds. This should be
  364. # made prettier.
  365. #
  366. my $document = $dom->createElement("document");
  367. $document->setAttribute("language", $lang);
  368. $root->appendChild($document);
  369. my $source = "$opts{i}/$file.$lang.xhtml";
  370. unless (-f $source) {
  371. my $missingsource = $source;
  372. if (-f "$opts{i}/$file.en.xhtml") {
  373. $document->setAttribute("language", "en");
  374. $source = "$opts{i}/$file.en.xhtml";
  375. } elsif (-f "$opts{i}/$file.".$root->getAttribute("original").".xhtml") {
  376. $document->setAttribute("language", $root->getAttribute("original"));
  377. $source = "$opts{i}/$file.".$root->getAttribute("original").".xhtml";
  378. } else {
  379. my $l = (keys %{$bases{$file}})[0];
  380. $document->setAttribute("language", $l);
  381. $source = "$opts{i}/$file.$l.xhtml";
  382. }
  383. if ($dir eq "global") {
  384. lock(*TRANSLATIONS);
  385. print TRANSLATIONS "$lang $missingsource $source\n";
  386. unlock(*TRANSLATIONS);
  387. }
  388. }
  389. if ( (stat("$opts{o}/$dir/$file.$lang.html"))[9] >
  390. (stat($source))[9] && $opts{u} && ! -f "$opts{i}/$file.xsl" ) {
  391. next;
  392. }
  393. #
  394. # Here begins automated magic for those pages which we need to
  395. # assemble other sets of informations for first (automatically
  396. # updated pages).
  397. #
  398. if (-f "$opts{i}/$file.xsl") {
  399. #
  400. # Settle down please, children. First we remove all previous
  401. # document leftovers.
  402. #
  403. foreach ($root->getElementsByTagName("document")) {
  404. $root->removeChild($_);
  405. }
  406. $root->appendChild($document);
  407. # Create the <timestamp> tag automatically for these documents
  408. my $timestamp = $dom->createElement("timestamp");
  409. $timestamp->appendText("\$"."Date: ".$current_time." \$ \$"."Author: automatic \$");
  410. $document->appendChild($timestamp);
  411. #
  412. # Get the list of sources and create the files hash. The files
  413. # hash contains the base name for each file we want to use, and
  414. # then the language for it as a value. It prefers a file in the
  415. # language we're building into, but will accept an English file as
  416. # a substitute.
  417. #
  418. # "Learn all that is learnable and return that information
  419. # to the Creator."
  420. #
  421. open(IN, '<', "$opts{i}/$file.sources");
  422. my @auto_sources = <IN>;
  423. close IN;
  424. my %files;
  425. foreach (@auto_sources) {
  426. if (/(.*):[a-z,]*global/ || /(.*):[a-z,]*$dir/) {
  427. foreach my $f (glob("$1*")) {
  428. if ($f =~ /(.*)\.([a-z][a-z])\.xml$/) {
  429. if (!$files{$1}) {
  430. $files{$1} = $2;
  431. } elsif ($2 eq $lang) {
  432. $files{$1} = $2;
  433. } elsif (($2 eq "en") &&
  434. ($files{$1} ne $lang)) {
  435. $files{$1} = $2;
  436. }
  437. }
  438. }
  439. }
  440. }
  441. #
  442. # With that information, we load the source document and create
  443. # a new element in it, called <set>, which will hold the combined
  444. # knowledge of all the sets in the source files.
  445. #
  446. my $sourcedoc = $parser->parse_file($source);
  447. $sourcedoc->documentElement->setAttribute("date", $current_date);
  448. $sourcedoc->documentElement->setAttribute("lang", $lang);
  449. my $auto_data = $sourcedoc->createElement("set");
  450. while (my ($base, $l) = each %files) {
  451. if (($dir eq "global") && ($l ne $lang)) {
  452. lock(*TRANSLATIONS);
  453. print TRANSLATIONS "$lang $base.$lang.xml $base.$l.xml\n";
  454. unlock(*TRANSLATIONS);
  455. }
  456. print STDERR "Loading $base.$l.xml\n" if $opts{d};
  457. my $source_data = $parser->parse_file("$base.$l.xml");
  458. foreach ($source_data->documentElement->childNodes) {
  459. my $c = $_->cloneNode(1);
  460. # add the filename to nodes (news, events, …) so that we can use it as an identifier (e.g. for RSS)
  461. if (ref($c) eq "XML::LibXML::Element") {
  462. $base =~ /.*[\/_]([^\/_]*$)/;
  463. $c->setAttribute( "filename", $1 );
  464. }
  465. $auto_data->appendChild($c);
  466. }
  467. }
  468. $sourcedoc->documentElement->appendChild($auto_data);
  469. #
  470. # Get the appropriate textset for this language. If one can't be
  471. # found, use the English. (I hope this never happens)
  472. #
  473. my $textlang = $lang;
  474. unless (-f $opts{i}."/tools/texts-content-$textlang.xml") {
  475. $textlang = "en";
  476. }
  477. my $textdoc = $sourcedoc->createElement("textset-content");
  478. $auto_data->appendChild($textdoc);
  479. clone_document($textdoc, $opts{i}."/tools/texts-content-$textlang.xml");
  480. # Get also backup texts from the English file
  481. my $textdocbak = $sourcedoc->createElement("textset-content-backup");
  482. $auto_data->appendChild($textdocbak);
  483. clone_document($textdocbak, $opts{i}."/tools/texts-content-en.xml");
  484. # TODO: optimise getting texts-content-xx.xml and texts-content-en.xml,
  485. # since it does not depend on the xsl file being treated, we should do it only once!
  486. #
  487. # Transform the document using the XSL file and then push the
  488. # result into the <document> element of the document we're building.
  489. #
  490. my $style_doc = $parser->parse_file("$opts{i}/$file.xsl");
  491. my $stylesheet = $xslt_parser->parse_stylesheet($style_doc);
  492. my $results = $stylesheet->transform($sourcedoc);
  493. foreach ($results->documentElement->childNodes) {
  494. my $c = $_->cloneNode(1);
  495. $document->appendChild($c);
  496. }
  497. #
  498. # Now, while we're just at it, we create the RSS feeds if we want any
  499. #
  500. if (-f "$opts{i}/$file.rss.xsl") {
  501. my $style_doc = $parser->parse_file("$opts{i}/$file.rss.xsl");
  502. my $stylesheet = $xslt_parser->parse_stylesheet($style_doc);
  503. my $results = $stylesheet->transform($sourcedoc);
  504. $stylesheet->output_file($results, "$opts{o}/$dir/$file.$lang.rss")
  505. unless $opts{n};
  506. }
  507. #
  508. # and possibly the corresponding iCal (ics) file
  509. #
  510. if (-f "$opts{i}/$file.ics.xsl") {
  511. my $style_doc = $parser->parse_file("$opts{i}/$file.ics.xsl");
  512. my $stylesheet = $xslt_parser->parse_stylesheet($style_doc);
  513. my $results = $stylesheet->transform($sourcedoc);
  514. $stylesheet->output_file($results, "$opts{o}/$dir/$file.$lang.ics")
  515. unless $opts{n};
  516. }
  517. } else {
  518. #
  519. # If this wasn't an automatically updating document, we simply
  520. # clone the contents of the source file into the document.
  521. #
  522. clone_document($document, $source);
  523. }
  524. #
  525. # Find out if this translation is to be regarded as outdated or not.
  526. # A translation is deemed outdated if it is more than 2 hours older
  527. # than the original. This makes sure a translation committed together
  528. # with the original (but maybe a second earlier) isn't marked outdated.
  529. #
  530. my $originalsource = "$file.".$root->getAttribute("original").".xhtml";
  531. my $comment;
  532. my $old_outdated = 0;
  533. if (( stat("$opts{i}/$originalsource"))[9] > (stat($source))[9] + 7200
  534. and not $cant_be_outdated{$file} ) {
  535. $root->setAttribute("outdated", "yes");
  536. if ($dir eq "global") {
  537. lock(*TRANSLATIONS);
  538. print TRANSLATIONS "$lang $source $originalsource\n";
  539. unlock(*TRANSLATIONS);
  540. }
  541. } else {
  542. $root->setAttribute("outdated", "no");
  543. }
  544. my $textdoc = $dom->createElement("textset");
  545. $root->appendChild($textdoc);
  546. clone_document($textdoc, $opts{i}."/tools/texts-$textlang.xml");
  547. my $textdoc = $dom->createElement("textset");
  548. $root->appendChild($textdoc);
  549. clone_document($textdoc, $opts{i}."/tools/texts-$textlang.xml");
  550. #
  551. # Read the fundraising text, if it exists.
  552. #
  553. if (-f $opts{i}."/fundraising.$lang.xml") {
  554. my $fundraisingdoc = $dom->createElement("fundraising");
  555. $root->appendChild($fundraisingdoc);
  556. clone_document($fundraisingdoc, $opts{i}."/fundraising.$lang.xml");
  557. } elsif (-f $opts{i}."/fundraising.en.xml") {
  558. my $fundraisingdoc = $dom->createElement("fundraising");
  559. $root->appendChild($fundraisingdoc);
  560. clone_document($fundraisingdoc, $opts{i}."/fundraising.en.xml");
  561. }
  562. #
  563. # Read the fundraising text, if it exists.
  564. #
  565. if (-f $opts{i}."/fundraising.$lang.xml") {
  566. my $fundraisingdoc = $dom->createElement("fundraising");
  567. $root->appendChild($fundraisingdoc);
  568. clone_document($fundraisingdoc, $opts{i}."/fundraising.$lang.xml");
  569. } elsif (-f $opts{i}."/fundraising.en.xml") {
  570. my $fundraisingdoc = $dom->createElement("fundraising");
  571. $root->appendChild($fundraisingdoc);
  572. clone_document($fundraisingdoc, $opts{i}."/fundraising.en.xml");
  573. }
  574. #
  575. # And then we do the same thing for the menues. But first we take the
  576. # global menu here, then we add any information that is specific to
  577. # the focus.
  578. #
  579. foreach ($root->getElementsByTagName("menuset")) {
  580. $root->removeChild($_);
  581. }
  582. #
  583. # And then we do the same thing for the menues. But first we take the
  584. # global menu here, then we add any information that is specific to
  585. # the focus.
  586. #
  587. foreach ($root->getElementsByTagName("menuset")) {
  588. $root->removeChild($_);
  589. }
  590. my %menu;
  591. foreach ('global', $dir) {
  592. if (-f $opts{i}."/tools/menu-$_.xml") {
  593. my $menudoc = $parser->parse_file($opts{i}."/tools/menu-$_.xml");
  594. foreach my $n ($menudoc->documentElement->getElementsByTagName("menu")) {
  595. $menu{$n->getAttribute("id")} = $n;
  596. }
  597. }
  598. }
  599. my $menuroot = $dom->createElement("menuset");
  600. while (my ($id, $n) = each %menu) {
  601. my $m = $n->cloneNode(1);
  602. $menuroot->appendChild($m);
  603. }
  604. $root->appendChild($menuroot);
  605. # <start addendum> (TODO: transform this into a function)
  606. #
  607. # Get the list of sources and create the files hash. The files
  608. # hash contains the base name for each file we want to use, and
  609. # then the language for it as a value. It prefers a file in the
  610. # language we're building into, but will accept an English file as
  611. # a substitute.
  612. #
  613. # "Learn all that is learnable and return that information
  614. # to the Creator."
  615. #
  616. open(IN, '<', "$opts{i}/fsfe.sources");
  617. my @auto_sources = <IN>;
  618. close IN;
  619. my %files;
  620. foreach (@auto_sources) {
  621. if (/(.*):[a-z,]*global/ || /(.*):[a-z,]*$dir/) {
  622. foreach my $f (glob("$1*")) {
  623. if ($f =~ /(.*)\.([a-z][a-z])\.xml$/) {
  624. if (!$files{$1}) {
  625. $files{$1} = $2;
  626. } elsif ($2 eq $lang) {
  627. $files{$1} = $2;
  628. } elsif (($2 eq "en") &&
  629. ($files{$1} ne $lang)) {
  630. $files{$1} = $2;
  631. }
  632. }
  633. }
  634. }
  635. }
  636. #
  637. # With that information, we load the source document and create
  638. # a new element in it, called <set>, which will hold the combined
  639. # knowledge of all the sets in the source files.
  640. #
  641. foreach ($root->getElementsByTagName("set")) {
  642. $root->removeChild($_);
  643. }
  644. my $auto_data = $dom->createElement("set");
  645. while (my ($base, $l) = each %files) {
  646. if (($dir eq "global") && ($l ne $lang)) {
  647. lock(*TRANSLATIONS);
  648. print TRANSLATIONS "$lang $base.$lang.xml $base.$l.xml\n";
  649. unlock(*TRANSLATIONS);
  650. }
  651. print STDERR "Loading $base.$l.xml\n" if $opts{d};
  652. my $source_data = $parser->parse_file("$base.$l.xml");
  653. foreach ($source_data->documentElement->childNodes) {
  654. my $c = $_->cloneNode(1);
  655. # add the filename to nodes (news, events, …) so that we can use it as an identifier (e.g. for RSS)
  656. if (ref($c) eq "XML::LibXML::Element") {
  657. $base =~ /.*[\/_]([^\/_]*$)/;
  658. $c->setAttribute( "filename", $1 );
  659. }
  660. $auto_data->appendChild($c);
  661. }
  662. }
  663. $root->appendChild($auto_data);
  664. # <end addendum>
  665. if ( $lang eq "fr" and $file eq $file_to_treat ) {
  666. print "--->outputting test2.xml\n";
  667. open (TEST, '>', "/home/nicolas/FSFE/fsfe-web-out/test2.xml");
  668. print TEST $dom->toString();
  669. close (TEST);
  670. }
  671. #
  672. # Do the actual transformation. (through fsfe.xsl)
  673. #
  674. my $results = $global_stylesheet->transform($dom);
  675. #
  676. # In post-processing, we replace links pointing back to ourselves
  677. # so that they point to the correct language.
  678. #
  679. foreach ($results->documentElement->getElementsByTagName("a")) {
  680. my $href = $_->getAttribute("href");
  681. if ($href =~ /^http:\/\/www.fsfe.org/) {
  682. if ($_->textContent != "Our global work") {
  683. $href =~ s/http:\/\/www.fsfe.org//;
  684. }
  685. }
  686. if (($href !~ /^http/) && ($href !~ /^#/)) {
  687. # Save possible anchor and remove it from URL
  688. my $anchor = $href;
  689. if (!($anchor =~ s/.*#/#/)) {
  690. $anchor = "";
  691. }
  692. $href =~ s/#.*//;
  693. # proces URL
  694. if (($href =~ /\.html$/) && ($href !~ /\.[a-z][a-z]\.html$/)) {
  695. $href =~ s/\.html$/\.$lang.html/;
  696. } elsif (($href =~ /\.rss$/) && ($href !~ /\.[a-z][a-z]\.rss$/)) {
  697. $href =~ s/\.rss$/\.$lang.rss/;
  698. } elsif (($href =~ /\.ics$/) && ($href !~ /\.[a-z][a-z]\.ics$/)) {
  699. $href =~ s/\.ics$/\.$lang.ics/;
  700. } else {
  701. if (-d $opts{i}."/$href") {
  702. $href =~ s/\/?$/\/index.$lang.html/;
  703. } elsif ($href =~ /\/\w+$/) {
  704. $href .= ".$lang.html";
  705. }
  706. }
  707. # replace anchor
  708. $href .= $anchor;
  709. # For pages running on an external server, use full URL
  710. if ($document->getAttribute("external")) {
  711. $href = "http://www.fsfe.org$href";
  712. }
  713. $_->setAttribute("href", $href);
  714. }
  715. }
  716. print "Writing: $opts{o}/$dir/$file.$lang.html\n" if $opts{d};
  717. $global_stylesheet->output_file($results, "$opts{o}/$dir/$file.$lang.html")
  718. unless $opts{n};
  719. # Add foo.html.xx link which is used by Apache's MultiViews option when
  720. # a user enters foo.html as URL.
  721. link("$opts{o}/$dir/$file.$lang.html", "$opts{o}/$dir/$file.html.$lang")
  722. unless $opts{n};
  723. }
  724. }
  725. print STDERR "\n" unless $opts{q};
  726. }
  727. # Close the logfile for outdated and missing translations
  728. close (TRANSLATIONS);
  729. print STDERR "Fixing index links\n" unless $opts{q};
  730. while (my ($path, undef) = each %countries) {
  731. my @dirs = File::Find::Rule->directory()
  732. ->in($opts{o}."/$path");
  733. foreach (@dirs) {
  734. my $base = basename($_);
  735. while (my ($lang, undef) = each %languages) {
  736. if (-f "$_/$base.$lang.html" &&
  737. ! -f "$_/index.$lang.html") {
  738. link("$_/$base.$lang.html", "$_/index.$lang.html")
  739. unless $opts{n};
  740. link("$_/$base.html.$lang", "$_/index.html.$lang")
  741. unless $opts{n};
  742. }
  743. }
  744. }
  745. }
  746. #
  747. # For all files that are not XHTML source files, we copy them verbatim to
  748. # the final location, for each focus. These should be links instead to
  749. # prevent us from wasting disk space.
  750. #
  751. print STDERR "Copying misc files\n" unless $opts{q};
  752. foreach (grep(!/\.sources$/, grep(!/\.xsl$/, grep(!/\.xml$/, grep(!/\.xhtml$/,
  753. @files))))) {
  754. while (my ($dir, undef) = each %countries) {
  755. if (-f "$opts{i}/$_" && !$opts{n}) {
  756. link("$opts{i}/$_", "$opts{o}/$dir/$_");
  757. }
  758. }
  759. }
  760. #
  761. # Helper function that clones a document. It accepts an XML node and
  762. # a filename as parameters. Using this, it loads the source file into
  763. # the XML node.
  764. #
  765. sub clone_document {
  766. my ($doc, $source) = @_;
  767. my $root = $doc->parentNode;
  768. print "Source: $source\n" if $opts{d};
  769. foreach ($root->getElementsByTagName($doc->nodeName)) {
  770. $root->removeChild($_);
  771. }
  772. $root->appendChild($doc);
  773. my $parser = XML::LibXML->new('encoding'=>'utf-8');
  774. $parser->load_ext_dtd(0);
  775. $parser->recover(1);
  776. my $sourcedoc = $parser->parse_file($source);
  777. foreach ($sourcedoc->documentElement->childNodes) {
  778. $_->unbindNode();
  779. my $n = $_->cloneNode(1);
  780. $doc->appendChild($n);
  781. }
  782. if ($sourcedoc->documentElement->getAttribute("external")) {
  783. $doc->setAttribute("external", "yes");
  784. }
  785. if ($sourcedoc->documentElement->getAttribute("newsdate")) {
  786. # necessary for xhtml news files
  787. $doc->setAttribute("newsdate", $sourcedoc->documentElement->getAttribute("newsdate"));
  788. }
  789. if ($sourcedoc->documentElement->getAttribute("type")) {
  790. # necessary to differentiate news and newsletter pages
  791. # TODO: find a way to copy all such attributes!
  792. $doc->setAttribute("type", $sourcedoc->documentElement->getAttribute("type"));
  793. }
  794. }
  795. #
  796. # Helper functions to lock and unlock the translation log.
  797. #
  798. sub lock {
  799. my ($fh) = @_;
  800. flock($fh, LOCK_EX);
  801. seek($fh, 0, 2);
  802. }
  803. sub unlock {
  804. my ($fh) = @_;
  805. flock($fh, LOCK_UN);
  806. }