1 /**
2 	Internationalization/translation support for the web interface module.
3 
4 	Copyright: © 2014-2017 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig
7 */
8 module vibe.web.i18n;
9 
10 import vibe.http.server : HTTPServerRequest;
11 
12 import std.algorithm : canFind, min, startsWith;
13 import std.range.primitives : ElementType, isForwardRange, save;
14 import std.range : only;
15 
16 /**
17 	Annotates an interface method or class with translation information.
18 
19 	The translation context contains information about supported languages
20 	and the translated strings. Any translations will be automatically
21 	applied to Diet templates, as well as strings passed to
22 	$(D vibe.web.web.trWeb).
23 
24 	By default, the "Accept-Language" header of the incoming request will be
25 	used to determine the language used. To override this behavior, add a
26 	static method $(D determineLanguage) to the translation context, which
27 	takes the request and returns a language string (see also the second
28 	example).
29 */
30 @property TranslationContextAttribute!CONTEXT translationContext(CONTEXT)() { return TranslationContextAttribute!CONTEXT.init; }
31 
32 ///
33 unittest {
34 	import vibe.http.router : URLRouter;
35 	import vibe.web.web : registerWebInterface;
36 
37 	struct TranslationContext {
38 		import std.typetuple;
39 		alias languages = TypeTuple!("en_US", "de_DE", "fr_FR");
40 		//mixin translationModule!"app";
41 		//mixin translationModule!"somelib";
42 	}
43 
44 	@translationContext!TranslationContext
45 	class MyWebInterface {
46 		void getHome()
47 		{
48 			//render!("home.dt")
49 		}
50 	}
51 
52 	void test(URLRouter router)
53 	{
54 		router.registerWebInterface(new MyWebInterface);
55 	}
56 }
57 
58 /// Defining a custom function for determining the language.
59 unittest {
60 	import vibe.http.router : URLRouter;
61 	import vibe.http.server;
62 	import vibe.web.web : registerWebInterface;
63 
64 	struct TranslationContext {
65 		import std.typetuple;
66 		// A language can be in the form en_US, en-US or en. Put the languages you want to prioritize first.
67 		alias languages = TypeTuple!("en_US", "de_DE", "fr_FR");
68 		//mixin translationModule!"app";
69 		//mixin translationModule!"somelib";
70 
71 		// use language settings from the session instead of using the
72 		// "Accept-Language" header
73 		static string determineLanguage(scope HTTPServerRequest req)
74 		{
75 			if (!req.session) return req.determineLanguageByHeader(languages); // default behaviour using "Accept-Language" header
76 			return req.session.get("language", "");
77 		}
78 	}
79 
80 	@translationContext!TranslationContext
81 	class MyWebInterface {
82 		void getHome()
83 		{
84 			//render!("home.dt")
85 		}
86 	}
87 
88 	void test(URLRouter router)
89 	{
90 		router.registerWebInterface(new MyWebInterface);
91 	}
92 }
93 
94 @safe unittest {
95 	import vibe.http.router : URLRouter;
96 	import vibe.http.server : HTTPServerRequest;
97 	import vibe.web.web : registerWebInterface;
98 
99 	struct TranslationContext {
100 		import std.typetuple;
101 		alias languages = TypeTuple!("en_US", "de_DE", "fr_FR");
102 		static string determineLanguage(scope HTTPServerRequest req) { return "en_US"; }
103 	}
104 
105 	@translationContext!TranslationContext
106 	class MyWebInterface { void getHome() @safe {} }
107 
108 	auto router = new URLRouter;
109 	router.registerWebInterface(new MyWebInterface);
110 }
111 
112 
113 struct TranslationContextAttribute(CONTEXT) {
114 	alias Context = CONTEXT;
115 }
116 
117 /*
118 doctype 5
119 html
120 	body
121 		p& Hello, World!
122 		p& This is a translated version of #{appname}.
123 	html
124 		p(class="Sasdasd")&.
125 			This is a complete paragraph of translated text.
126 */
127 
128 /** Makes a set of PO files available to a web interface class.
129 
130 	This mixin template needs to be mixed in at the class scope. It will parse all
131 	translation files with the specified file name prefix and make their
132 	translations available.
133 
134 	Params:
135 		FILENAME = Base name of the set of PO files to mix in. A file with the
136 			name `"<FILENAME>.<LANGUAGE>.po"` must be available as a string import
137 			for each language defined in the translation context.
138 
139 	Bugs:
140 		`FILENAME` should not contain (back)slash characters, as string imports
141 		from sub directories will currently fail on Windows. See
142 		$(LINK https://issues.dlang.org/show_bug.cgi?id=14349).
143 
144 	See_Also: `translationContext`
145 */
146 mixin template translationModule(string FILENAME)
147 {
148 	import std.string : tr;
149 	enum NAME = FILENAME.tr(`/.-\`, "____");
150 	private static string file_mixins() {
151 		string ret;
152 		foreach (language; languages)
153 			ret ~= "enum "~language~"_"~NAME~" = extractDeclStrings(import(`"~FILENAME~"."~language~".po`));\n";
154 		return ret;
155 	}
156 
157 	mixin(file_mixins);
158 }
159 
160 template languageSeq(CTX) {
161 	static if (is(typeof([CTX.languages]) : string[])) alias languageSeq = CTX.languages;
162 	else alias languageSeq = aliasSeqOf!(CTX.languages);
163 }
164 
165 /**
166 	Performs the string translation for a statically given language.
167 
168 	The second overload takes a plural form and a number to select from a set
169 	of translations based on the plural forms of the target language.
170 */
171 template tr(CTX, string LANG)
172 {
173 	string tr(string key, string context = null)
174 	{
175 		return tr!(CTX, LANG)(key, null, 0, context);
176 	}
177 
178 	string tr(string key, string key_plural, int n, string context = null)
179 	{
180 		static assert([languageSeq!CTX].canFind(LANG), "Unknown language: "~LANG);
181 
182 		foreach (i, mname; __traits(allMembers, CTX)) {
183 			static if (mname.startsWith(LANG~"_")) {
184 				enum langComponents = __traits(getMember, CTX, mname);
185 				foreach (entry; langComponents.messages) {
186 					if ((context is null) == (entry.context is null)) {
187 						if (context is null || entry.context == context) {
188 							if (entry.key == key) {
189 								if (key_plural !is null) {
190 									if (entry.pluralKey !is null && entry.pluralKey == key_plural) {
191 										static if (langComponents.nplurals_expr !is null && langComponents.plural_func_expr !is null) {
192 											mixin("int nplurals = "~langComponents.nplurals_expr~";");
193 											if (nplurals > 0) {
194 												mixin("int index = "~langComponents.plural_func_expr~";");
195 												return entry.pluralValues[index];
196 											}
197 											return entry.value;
198 										}
199 										assert(false, "Plural translations are not supported when the po file does not contain an entry for Plural-Forms.");
200 									}
201 								} else {
202 									return entry.value;
203 								}
204 							}
205 						}
206 					}
207 				}
208 			}
209 		}
210 
211 		static if (is(typeof(CTX.enforceExistingKeys)) && CTX.enforceExistingKeys) {
212 			if (key_plural !is null) {
213 				if (context is null) {
214 					assert(false, "Missing translation keys for "~LANG~": "~key~"&"~key_plural);
215 				}
216 				assert(false, "Missing translation key for "~LANG~"; "~context~": "~key~"&"~key_plural);
217 			}
218 
219 			if (context is null) {
220 				assert(false, "Missing translation key for "~LANG~": "~key);
221 			}
222 			assert(false, "Missing translation key for "~LANG~"; "~context~": "~key);
223 		} else {
224 			return n == 1 || !key_plural.length ? key : key_plural;
225 		}
226 	}
227 }
228 
229 /// Determines a language code from the value of a header string.
230 /// Returns: The best match from the Accept-Language header for a language. `null` if there is no supported language.
231 public string determineLanguageByHeader(T)(string accept_language, T allowed_languages) @safe pure @nogc
232 	if (isForwardRange!T && is(ElementType!T : string) || is(T == typeof(only())))
233 {
234 	import std.algorithm : splitter, countUntil;
235 	import std.string : indexOf;
236 
237 	// TODO: verify that allowed_languages doesn't contain a mix of languages with and without extra specifier for the same lanaguage (but only if one without specifier comes before those with specifier)
238 	// Implementing that feature should try to give a compile time warning and not change the behaviour of this function.
239 
240 	if (!accept_language.length)
241 		return null;
242 
243 	string fallback = null;
244 	foreach (accept; accept_language.splitter(",")) {
245 		auto sidx = accept.indexOf(';');
246 		if (sidx >= 0)
247 			accept = accept[0 .. sidx];
248 
249 		string alang, aextra;
250 		auto asep = accept.countUntil!(a => a == '_' || a == '-');
251 		if (asep < 0)
252 			alang = accept;
253 		else {
254 			alang = accept[0 .. asep];
255 			aextra = accept[asep + 1 .. $];
256 		}
257 
258 		static if (!is(T == typeof(only()))) { // workaround for type errors
259 			foreach (lang; allowed_languages.save) {
260 				string lcode, lextra;
261 				sidx = lang.countUntil!(a => a == '_' || a == '-');
262 				if (sidx < 0)
263 					lcode = lang;
264 				else {
265 					lcode = lang[0 .. sidx];
266 					lextra = lang[sidx + 1 .. $];
267 				}
268 				// request en_US == serve en_US
269 				if (lcode == alang && lextra == aextra)
270 					return lang;
271 				// request en_* == serve en
272 				if (lcode == alang && !lextra.length)
273 					return lang;
274 				// request en == serve en_*
275 				if (lcode == alang && !aextra.length)
276 					return lang;
277 				// request en* == serve en_* && be first occurence
278 				if (lcode == alang && lextra.length && !fallback.length)
279 					fallback = lang;
280 			}
281 		}
282 	}
283 
284 	return fallback;
285 }
286 
287 /// ditto
288 public string determineLanguageByHeader(Tuple...)(string accept_language, Tuple allowed_languages) @safe pure @nogc
289 	if (Tuple.length != 1 || is(Tuple[0] : string))
290 {
291 	return determineLanguageByHeader(accept_language, only(allowed_languages));
292 }
293 
294 /// ditto
295 public string determineLanguageByHeader(T)(HTTPServerRequest req, T allowed_languages) @safe pure
296 	if (isForwardRange!T && is(ElementType!T : string) || is(T == typeof(only())))
297 {
298 	return determineLanguageByHeader(req.headers.get("Accept-Language", null), allowed_languages);
299 }
300 
301 /// ditto
302 public string determineLanguageByHeader(Tuple...)(HTTPServerRequest req, Tuple allowed_languages) @safe pure
303 	if (Tuple.length != 1 || is(Tuple[0] : string))
304 {
305 	return determineLanguageByHeader(req.headers.get("Accept-Language", null), only(allowed_languages));
306 }
307 
308 @safe unittest {
309 	assert(determineLanguageByHeader("de-DE,de;q=0.8,en;q=0.6,en-US;q=0.4", ["en-US", "de_DE", "de_CH"]) == "de_DE");
310 	assert(determineLanguageByHeader("de-CH,de;q=0.8,en;q=0.6,en-US;q=0.4", ["en_US", "de_DE", "de-CH"]) == "de-CH");
311 	assert(determineLanguageByHeader("en_CA,en_US", ["ja_JP", "en"]) == "en");
312 	assert(determineLanguageByHeader("en", ["ja_JP", "en"]) == "en");
313 	assert(determineLanguageByHeader("en", ["ja_JP", "en_US"]) == "en_US");
314 	assert(determineLanguageByHeader("en_US", ["ja-JP", "en"]) == "en");
315 	assert(determineLanguageByHeader("de-DE,de;q=0.8,en;q=0.6,en-US;q=0.4", ["ja_JP"]) is null);
316 	assert(determineLanguageByHeader("de, de-DE ;q=0.8 , en ;q=0.6 , en-US;q=0.4", ["de-DE"]) == "de-DE");
317 	assert(determineLanguageByHeader("en_GB", ["en_US"]) == "en_US");
318 	assert(determineLanguageByHeader("de_DE", ["en_US"]) is null);
319 	assert(determineLanguageByHeader("en_US,enCA", ["en_GB"]) == "en_GB");
320 	assert(determineLanguageByHeader("en_US,enCA", ["en_GB", "en"]) == "en");
321 	assert(determineLanguageByHeader("en_US,enCA", ["en", "en_GB"]) == "en");
322 	assert(determineLanguageByHeader("de,en_US;q=0.8,en;q=0.6", ["en_US", "de_DE"]) == "de_DE");
323 	// TODO from above (should be invalid input having a more generic language first in the list!)
324 	//assert(determineLanguageByHeader("en_US,enCA", ["en", "en_US"]) == "en_US");
325 }
326 
327 package string determineLanguage(alias METHOD)(scope HTTPServerRequest req)
328 {
329 	alias CTX = GetTranslationContext!METHOD;
330 
331 	static if (!is(CTX == void)) {
332 		static if (is(typeof(CTX.determineLanguage(req)))) {
333 			static assert(is(typeof(CTX.determineLanguage(req)) == string),
334 				"determineLanguage in a translation context must return a language string.");
335 			return CTX.determineLanguage(req);
336 		} else {
337 			return determineLanguageByHeader(req, only(CTX.languages));
338 		}
339 	} else return null;
340 }
341 
342 unittest { // make sure that the custom determineLanguage is called
343 	static struct CTX {
344 		static string determineLanguage(HTTPServerRequest a) { return "test"; }
345 	}
346 	@translationContext!CTX
347 	static class Test {
348 		void test()
349 		{
350 		}
351 	}
352 	auto test = new Test;
353 	assert(determineLanguage!(test.test)(HTTPServerRequest.init) == "test");
354 }
355 
356 unittest { // issue #1955
357 	import std.meta : AliasSeq;
358 	import vibe.inet.url : URL;
359 	import vibe.http.server : createTestHTTPServerRequest;
360 
361 	static struct CTX {
362 		alias languages = AliasSeq!();
363 	}
364 
365 	@translationContext!CTX
366 	class C {
367 		void test() {}
368 	}
369 
370 	auto req = createTestHTTPServerRequest(URL("http://127.0.0.1/test"));
371 	assert(determineLanguage!(C.test)(req) == null);
372 }
373 
374 package template GetTranslationContext(alias METHOD)
375 {
376 	import vibe.internal.meta.uda;
377 
378 	alias PARENT = typeof(__traits(parent, METHOD).init);
379 	enum FUNCTRANS = findFirstUDA!(TranslationContextAttribute, METHOD);
380 	enum PARENTTRANS = findFirstUDA!(TranslationContextAttribute, PARENT);
381 	static if (FUNCTRANS.found) alias GetTranslationContext = FUNCTRANS.value.Context;
382 	else static if (PARENTTRANS.found) alias GetTranslationContext = PARENTTRANS.value.Context;
383 	else alias GetTranslationContext = void;
384 }
385 
386 
387 private struct DeclString {
388 	string context;
389 	string key;
390 	string pluralKey;
391 	string value;
392 	string[] pluralValues;
393 }
394 
395 private struct LangComponents {
396 	DeclString[] messages;
397 	string nplurals_expr;
398 	string plural_func_expr;
399 }
400 
401 // Example po header
402 /*
403  * # Translation of kstars.po into Spanish.
404  * # This file is distributed under the same license as the kdeedu package.
405  * # Pablo de Vicente <pablo@foo.com>, 2005, 2006, 2007, 2008.
406  * # Eloy Cuadra <eloy@bar.net>, 2007, 2008.
407  * msgid ""
408  * msgstr ""
409  * "Project-Id-Version: kstars\n"
410  * "Report-Msgid-Bugs-To: http://bugs.kde.org\n"
411  * "POT-Creation-Date: 2008-09-01 09:37+0200\n"
412  * "PO-Revision-Date: 2008-07-22 18:13+0200\n"
413  * "Last-Translator: Eloy Cuadra <eloy@bar.net>\n"
414  * "Language-Team: Spanish <kde-l10n-es@kde.org>\n"
415  * "MIME-Version: 1.0\n"
416  * "Content-Type: text/plain; charset=UTF-8\n"
417  * "Content-Transfer-Encoding: 8bit\n"
418  * "Plural-Forms: nplurals=2; plural=n != 1;\n"
419  */
420 
421 // PO format notes
422 /*
423  * # - Translator comment
424  * #: - source reference
425  * #. - extracted comments
426  * #, - flags such as "c-format" to indicate what king of substitutions may be present
427  * #| msgid - previous string comment
428  * #~ - obsolete message
429  * msgctxt - disabmbiguating context, like variable scope (optional, defaults to null)
430  * msgid - key to translate from (required)
431  * msgid_plural - plural form of the msg id (optional)
432  * msgstr - value to translate to (required)
433  * msgstr[0] - indexed translation for handling the various plural forms
434  * msgstr[1] - ditto
435  * msgstr[2] - ditto and etc...
436  */
437 
438 LangComponents extractDeclStrings(string text)
439 {
440 	DeclString[] declStrings;
441 	string nplurals_expr;
442 	string plural_func_expr;
443 
444 	size_t i = 0;
445 	while (true) {
446 		i = skipToDirective(i, text);
447 		if (i >= text.length) break;
448 
449 		string context = null;
450 
451 		// msgctxt is an optional field
452 		if (text.length - i >= 7 && text[i .. i+7] == "msgctxt") {
453 			i = skipWhitespace(i+7, text);
454 
455 			auto icntxt = skipString(i, text);
456 			context = dstringUnescape(wrapText(text[i+1 .. icntxt-1]));
457 			i = skipToDirective(icntxt, text);
458 		}
459 
460 		// msgid is a required field
461 		assert(text.length - i >= 5 && text[i .. i+5] == "msgid", "Expected 'msgid', got '"~text[i .. min(i+10, $)]~"'.");
462 		i += 5;
463 
464 		i = skipWhitespace(i, text);
465 
466 		auto iknext = skipString(i, text);
467 		auto key = dstringUnescape(wrapText(text[i+1 .. iknext-1]));
468 		i = iknext;
469 
470 		i = skipToDirective(i, text);
471 
472 		// msgid_plural is an optional field
473 		string key_plural = null;
474 		if (text.length - i >= 12 && text[i .. i+12] == "msgid_plural") {
475 			i = skipWhitespace(i+12, text);
476 			auto iprl = skipString(i, text);
477 			key_plural = dstringUnescape(wrapText(text[i+1 .. iprl-1]));
478 			i = skipToDirective(iprl, text);
479 		}
480 
481 		// msgstr is a required field
482 		assert(text.length - i >= 6 && text[i .. i+6] == "msgstr", "Expected 'msgstr', got '"~text[i .. min(i+10, $)]~"'.");
483 		i += 6;
484 
485 		i = skipWhitespace(i, text);
486 		auto ivnext = skipString(i, text);
487 		auto value = dstringUnescape(wrapText(text[i+1 .. ivnext-1]));
488 		i = ivnext;
489 		i = skipToDirective(i, text);
490 
491 		// msgstr[n] is a required field when msgid_plural is not null, and ignored otherwise
492 		string[] value_plural;
493 		if (key_plural !is null) {
494 			while (text.length - i >= 6 && text[i .. i+6] == "msgstr") {
495 				i = skipIndex(i+6, text);
496 				i = skipWhitespace(i, text);
497 				auto ims = skipString(i, text);
498 
499 				string plural = dstringUnescape(wrapText(text[i+1 .. ims-1]));
500 				i = skipLine(ims, text);
501 
502 				// Is it safe to assume that the entries are always sequential?
503 				value_plural ~= plural;
504 			}
505 		}
506 
507 		// Add the translation for the current language
508 		if (key == "") {
509 			nplurals_expr = parse_nplurals(value);
510 			plural_func_expr = parse_plural_expression(value);
511 		}
512 
513 		declStrings ~= DeclString(context, key, key_plural, value, value_plural);
514 	}
515 
516 	return LangComponents(declStrings, nplurals_expr, plural_func_expr);
517 }
518 
519 // Verify that two simple messages can be read and parsed correctly
520 unittest {
521 	auto str = `
522 # first string
523 msgid "ordinal.1"
524 msgstr "first"
525 
526 # second string
527 msgid "ordinal.2"
528 msgstr "second"`;
529 
530 	auto components = extractDeclStrings(str);
531 	auto ds = components.messages;
532 	assert(2 == ds.length, "Not enough DeclStrings have been processed");
533 	assert(ds[0].key == "ordinal.1", "The first key is not right.");
534 	assert(ds[0].value == "first", "The first value is not right.");
535 	assert(ds[1].key == "ordinal.2", "The second key is not right.");
536 	assert(ds[1].value == "second", "The second value is not right.");
537 }
538 
539 // Verify that the fields cannot be defined out of order
540 unittest {
541 	import core.exception : AssertError;
542 	import std.exception : assertThrown;
543 
544 	auto str1 = `
545 # unexpected field ahead
546 msgstr "world"
547 msgid "hello"`;
548 
549 	assertThrown!AssertError(extractDeclStrings(str1));
550 }
551 
552 // Verify that string wrapping is handled correctly
553 unittest {
554 	auto str = `
555 # The following text is wrapped
556 msgid ""
557 "This is an example of text that "
558 "has been wrapped on two lines."
559 msgstr ""
560 "It should not matter where it takes place, "
561 "the strings should all be concatenated properly."`;
562 
563 	auto ds = extractDeclStrings(str).messages;
564 	assert(1 == ds.length, "Expected one DeclString to have been processed.");
565 	assert(ds[0].key == "This is an example of text that has been wrapped on two lines.", "Failed to properly wrap the key");
566 	assert(ds[0].value == "It should not matter where it takes place, the strings should all be concatenated properly.", "Failed to properly wrap the key");
567 }
568 
569 // Verify that string wrapping and unescaping is handled correctly on example of PO headers
570 unittest {
571 	auto str = `
572 # English translations for ThermoWebUI package.
573 # This file is put in the public domain.
574 # Automatically generated, 2015.
575 #
576 msgid ""
577 msgstr ""
578 "Project-Id-Version: PROJECT VERSION\n"
579 "Report-Msgid-Bugs-To: developer@example.com\n"
580 "POT-Creation-Date: 2015-04-13 17:55+0600\n"
581 "PO-Revision-Date: 2015-04-13 14:13+0600\n"
582 "Last-Translator: Automatically generated\n"
583 "Language-Team: none\n"
584 "Language: en\n"
585 "MIME-Version: 1.0\n"
586 "Content-Type: text/plain; charset=UTF-8\n"
587 "Content-Transfer-Encoding: 8bit\n"
588 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
589 `;
590 	auto expected = `Project-Id-Version: PROJECT VERSION
591 Report-Msgid-Bugs-To: developer@example.com
592 POT-Creation-Date: 2015-04-13 17:55+0600
593 PO-Revision-Date: 2015-04-13 14:13+0600
594 Last-Translator: Automatically generated
595 Language-Team: none
596 Language: en
597 MIME-Version: 1.0
598 Content-Type: text/plain; charset=UTF-8
599 Content-Transfer-Encoding: 8bit
600 Plural-Forms: nplurals=2; plural=(n != 1);
601 `;
602 
603 	auto ds = extractDeclStrings(str).messages;
604 	assert(1 == ds.length, "Expected one DeclString to have been processed.");
605 	assert(ds[0].key == "", "Failed to properly wrap or unescape the key");
606 	assert(ds[0].value == expected, "Failed to properly wrap or unescape the value");
607 }
608 
609 // Verify that the message context is properly parsed
610 unittest {
611 	auto str1 = `
612 # "C" is for cookie
613 msgctxt "food"
614 msgid "C"
615 msgstr "C is for cookie, that's good enough for me."`;
616 
617 	auto ds1 = extractDeclStrings(str1).messages;
618 	assert(1 == ds1.length, "Expected one DeclString to have been processed.");
619 	assert(ds1[0].context == "food", "Expected a context of food");
620 	assert(ds1[0].key == "C", "Expected to find the letter C for the msgid.");
621 	assert(ds1[0].value == "C is for cookie, that's good enough for me.", "Unexpected value encountered for the msgstr.");
622 
623 	auto str2 = `
624 # No context validation
625 msgid "alpha"
626 msgstr "First greek letter."`;
627 
628 	auto ds2 = extractDeclStrings(str2).messages;
629 	assert(1 == ds2.length, "Expected one DeclString to have been processed.");
630 	assert(ds2[0].context is null, "Expected the context to be null when it is not defined.");
631 }
632 
633 unittest {
634 	enum str = `
635 # "C" is for cookie
636 msgctxt "food"
637 msgid "C"
638 msgstr "C is for cookie, that's good enough for me."
639 
640 # "C" is for language
641 msgctxt "lang"
642 msgid "C"
643 msgstr "Catalan"
644 
645 # Just "C"
646 msgid "C"
647 msgstr "Third letter"
648 `;
649 
650 	enum components = extractDeclStrings(str);
651 
652 	struct TranslationContext {
653 		import std.typetuple;
654 		enum enforceExistingKeys = true;
655 		alias languages = TypeTuple!("en_US");
656 
657 		// Note that this is normally handled by mixing in an external file.
658 		enum en_US_unittest = components;
659 	}
660 
661 	auto newTr(string msgid, string msgcntxt = null) {
662 		return tr!(TranslationContext, "en_US")(msgid, msgcntxt);
663 	}
664 
665 	assert(newTr("C", "food") == "C is for cookie, that's good enough for me.", "Unexpected translation based on context.");
666 	assert(newTr("C", "lang") == "Catalan", "Unexpected translation based on context.");
667 	assert(newTr("C") == "Third letter", "Unexpected translation based on context.");
668 }
669 
670 unittest {
671 	enum str = `msgid ""
672 msgstr ""
673 "Project-Id-Version: kstars\\n"
674 "Plural-Forms: nplurals=2; plural=n != 1;\\n"
675 
676 msgid "One file was deleted."
677 msgid_plural "Files were deleted."
678 msgstr "One file was deleted."
679 msgstr[0] "1 file was deleted."
680 msgstr[1] "%d files were deleted."
681 
682 msgid "One file was created."
683 msgid_plural "Several files were created."
684 msgstr "One file was created."
685 msgstr[0] "1 file was created"
686 msgstr[1] "%d files were created."
687 `;
688 
689 	import std.stdio;
690 	enum components = extractDeclStrings(str);
691 
692 	struct TranslationContext {
693 		import std.typetuple;
694 		enum enforceExistingKeys = true;
695 		alias languages = TypeTuple!("en_US");
696 
697 		// Note that this is normally handled by mixing in an external file.
698 		enum en_US_unittest2 = components;
699 	}
700 
701 	auto newTr(string msgid, string msgid_plural, int count, string msgcntxt = null) {
702 		return tr!(TranslationContext, "en_US")(msgid, msgid_plural, count, msgcntxt);
703 	}
704 
705 	string expected = "1 file was deleted.";
706 	auto actual = newTr("One file was deleted.", "Files were deleted.", 1);
707 	assert(expected == actual, "Expected: '"~expected~"' but got '"~actual~"'");
708 
709 	expected = "%d files were deleted.";
710 	actual = newTr("One file was deleted.", "Files were deleted.", 42);
711 	assert(expected == actual, "Expected: '"~expected~"' but got '"~actual~"'");
712 }
713 
714 private size_t skipToDirective(size_t i, ref string text)
715 {
716 	while (i < text.length) {
717 		i = skipWhitespace(i, text);
718 		if (i < text.length && text[i] == '#') i = skipLine(i, text);
719 		else break;
720 	}
721 	return i;
722 }
723 
724 private size_t skipWhitespace(size_t i, ref string text)
725 {
726 	while (i < text.length && (text[i] == ' ' || text[i] == '\t' || text[i] == '\n' || text[i] == '\r'))
727 		i++;
728 	return i;
729 }
730 
731 private size_t skipLine(size_t i, ref string text)
732 {
733 	while (i < text.length && text[i] != '\r' && text[i] != '\n') i++;
734 	if (i+1 < text.length && (text[i+1] == '\r' || text[i+1] == '\n') && text[i] != text[i+1]) i++;
735 	return i+1;
736 }
737 
738 private size_t skipString(size_t i, ref string text)
739 {
740 	import std.conv : to;
741 	assert(text[i] == '"', "Expected to encounter the start of a string at position: "~to!string(i));
742 	i++;
743 	while (true) {
744 		assert(i < text.length, "Missing closing '\"' for string: "~text[i .. min($, 10)]);
745 		if (text[i] == '"') {
746 			if (i+1 < text.length) {
747 				auto j = skipWhitespace(i+1, text);
748 				if (j<text.length && text[j] == '"') return skipString(j, text);
749 			}
750 			return i+1;
751 		}
752 		if (text[i] == '\\') i += 2;
753 		else i++;
754 	}
755 }
756 
757 private size_t skipIndex(size_t i, ref string text) {
758 	import std.conv : to;
759 	assert(text[i] == '[', "Expected to encounter a plural form of msgstr at position: "~to!string(i));
760 	for (; i<text.length; ++i) {
761 		if (text[i] == ']') {
762 			return i+1;
763 		}
764 	}
765 	assert(false, "Missing a ']' for a msgstr in a translation file.");
766 }
767 
768 private string wrapText(string str)
769 {
770 	string ret;
771 	bool wrapped = false;
772 
773 	for (size_t i=0; i<str.length; ++i) {
774 		if (str[i] == '\\') {
775 			assert(i+1 < str.length, "The string ends with the escape char: " ~ str);
776 			ret ~= str[i..i+2];
777 			++i;
778 		} else if (str[i] == '"') {
779 			wrapped = true;
780 			size_t j = skipWhitespace(i+1, str);
781 			if (j < str.length && str[j] == '"') {
782 				i=j;
783 			}
784 		} else ret ~= str[i];
785 	}
786 
787 	if (wrapped) return ret;
788 	return str;
789 }
790 
791 private string parse_nplurals(string msgstr)
792 in { assert(msgstr, "An empty string cannot be parsed for Plural-Forms."); }
793 do {
794 	import std.string : indexOf, CaseSensitive;
795 
796 	auto start = msgstr.indexOf("Plural-Forms:", CaseSensitive.no);
797 	if (start > -1) {
798 		auto beg = msgstr.indexOf("nplurals=", start+13, CaseSensitive.no);
799 		if (beg > -1) {
800 			auto end = msgstr.indexOf(';', beg+9, CaseSensitive.no);
801 			if (end > -1) {
802 				return msgstr[beg+9 .. end];
803 			}
804 			return msgstr[beg+9 .. $];
805 		}
806 	}
807 
808 	return null;
809 }
810 
811 unittest {
812 	auto res = parse_nplurals("Plural-Forms: nplurals=2; plural=n != 1;\n");
813 	assert(res == "2", "Failed to parse the correct number of plural forms for a language.");
814 }
815 
816 private string parse_plural_expression(string msgstr)
817 in { assert(msgstr, "An empty string cannot be parsed for Plural-Forms."); }
818 do {
819 	import std.string : indexOf, CaseSensitive;
820 
821 	auto start = msgstr.indexOf("Plural-Forms:", CaseSensitive.no);
822 	if (start > -1) {
823 		auto beg = msgstr.indexOf("plural=", start+13, CaseSensitive.no);
824 		if (beg > -1) {
825 			auto end = msgstr.indexOf(';', beg+7, CaseSensitive.no);
826 			if (end > -1) {
827 				return msgstr[beg+7 .. end];
828 			}
829 			return msgstr[beg+7 .. $];
830 		}
831 	}
832 
833 	return null;
834 }
835 
836 unittest {
837 	auto res = parse_plural_expression("Plural-Forms: nplurals=2; plural=n != 1;\n");
838 	assert(res == "n != 1", "Failed to parse the plural expression for a language.");
839 }
840 
841 private string dstringUnescape(in string str)
842 {
843 	string ret;
844 	size_t i, start = 0;
845 	for( i = 0; i < str.length; i++ )
846 		if( str[i] == '\\' ){
847 			if( i > start ){
848 				if( start > 0 ) ret ~= str[start .. i];
849 				else ret = str[0 .. i];
850 			}
851 			assert(i+1 < str.length, "The string ends with the escape char: " ~ str);
852 			switch(str[i+1]){
853 				default: ret ~= str[i+1]; break;
854 				case 'r': ret ~= '\r'; break;
855 				case 'n': ret ~= '\n'; break;
856 				case 't': ret ~= '\t'; break;
857 			}
858 			i++;
859 			start = i+1;
860 		}
861 
862 	if( i > start ){
863 		if( start == 0 ) return str;
864 		else ret ~= str[start .. i];
865 	}
866 	return ret;
867 }