1 /**
2 	Internationalization/translation support for the web interface module.
3 
4 	Copyright: © 2014-2017 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig
7 */
8 module vibe.web.i18n;
9 
10 import vibe.http.server : HTTPServerRequest;
11 
12 import std.algorithm : canFind, min, startsWith;
13 import std.range.primitives : ElementType, isForwardRange, save;
14 import std.range : only;
15 
16 /**
17 	Annotates an interface method or class with translation information.
18 
19 	The translation context contains information about supported languages
20 	and the translated strings. Any translations will be automatically
21 	applied to Diet templates, as well as strings passed to
22 	$(D vibe.web.web.trWeb).
23 
24 	By default, the "Accept-Language" header of the incoming request will be
25 	used to determine the language used. To override this behavior, add a
26 	static method $(D determineLanguage) to the translation context, which
27 	takes the request and returns a language string (see also the second
28 	example).
29 */
30 @property TranslationContextAttribute!CONTEXT translationContext(CONTEXT)() { return TranslationContextAttribute!CONTEXT.init; }
31 
32 ///
33 unittest {
34 	import vibe.http.router : URLRouter;
35 	import vibe.web.web : registerWebInterface;
36 
37 	struct TranslationContext {
38 		import std.meta : AliasSeq;
39 		alias languages = AliasSeq!("en_US", "de_DE", "fr_FR");
40 		//mixin translationModule!"app";
41 		//mixin translationModule!"somelib";
42 	}
43 
44 	@translationContext!TranslationContext
45 	class MyWebInterface {
46 		void getHome()
47 		{
48 			//render!("home.dt")
49 		}
50 	}
51 
52 	void test(URLRouter router)
53 	{
54 		router.registerWebInterface(new MyWebInterface);
55 	}
56 }
57 
58 /// Defining a custom function for determining the language.
59 unittest {
60 	import vibe.http.router : URLRouter;
61 	import vibe.http.server;
62 	import vibe.web.web : registerWebInterface;
63 
64 	struct TranslationContext {
65 		import std.meta : AliasSeq;
66 		// A language can be in the form en_US, en-US or en. Put the languages you want to prioritize first.
67 		alias languages = AliasSeq!("en_US", "de_DE", "fr_FR");
68 		//mixin translationModule!"app";
69 		//mixin translationModule!"somelib";
70 
71 		// use language settings from the session instead of using the
72 		// "Accept-Language" header
73 		static string determineLanguage(scope HTTPServerRequest req)
74 		{
75 			if (!req.session) return req.determineLanguageByHeader(languages); // default behaviour using "Accept-Language" header
76 			return req.session.get("language", "");
77 		}
78 	}
79 
80 	@translationContext!TranslationContext
81 	class MyWebInterface {
82 		void getHome()
83 		{
84 			//render!("home.dt")
85 		}
86 	}
87 
88 	void test(URLRouter router)
89 	{
90 		router.registerWebInterface(new MyWebInterface);
91 	}
92 }
93 
94 @safe unittest {
95 	import vibe.http.router : URLRouter;
96 	import vibe.http.server : HTTPServerRequest;
97 	import vibe.web.web : registerWebInterface;
98 
99 	struct TranslationContext {
100 		import std.meta : AliasSeq;
101 		alias languages = AliasSeq!("en_US", "de_DE", "fr_FR");
102 		static string determineLanguage(scope HTTPServerRequest req) { return "en_US"; }
103 	}
104 
105 	@translationContext!TranslationContext
106 	class MyWebInterface { void getHome() @safe {} }
107 
108 	auto router = new URLRouter;
109 	router.registerWebInterface(new MyWebInterface);
110 }
111 
112 
113 struct TranslationContextAttribute(CONTEXT) {
114 	alias Context = CONTEXT;
115 }
116 
117 /*
118 doctype 5
119 html
120 	body
121 		p& Hello, World!
122 		p& This is a translated version of #{appname}.
123 	html
124 		p(class="Sasdasd")&.
125 			This is a complete paragraph of translated text.
126 */
127 
128 /** Makes a set of PO files available to a web interface class.
129 
130 	This mixin template needs to be mixed in at the class scope. It will parse all
131 	translation files with the specified file name prefix and make their
132 	translations available.
133 
134 	Params:
135 		FILENAME = Base name of the set of PO files to mix in. A file with the
136 			name `"<FILENAME>.<LANGUAGE>.po"` must be available as a string import
137 			for each language defined in the translation context.
138 
139 	Bugs:
140 		`FILENAME` should not contain (back)slash characters, as string imports
141 		from sub directories will currently fail on Windows. See
142 		$(LINK https://issues.dlang.org/show_bug.cgi?id=14349).
143 
144 	See_Also: `translationContext`
145 */
146 mixin template translationModule(string FILENAME, string language_separator = ".")
147 {
148 	static import std.string;
149 	enum NAME = std..string.tr(FILENAME, `/.\\-`, "____");
150 	private static string file_mixins() {
151 		string ret;
152 		foreach (language; languages)
153 			ret ~= "enum "~language~"_"~NAME~" = extractDeclStrings(import(`"~FILENAME~language_separator~language~".po`));\n";
154 		return ret;
155 	}
156 
157 	mixin(file_mixins);
158 }
159 
160 template languageSeq(CTX) {
161 	static if (is(typeof([CTX.languages]) : string[])) alias languageSeq = CTX.languages;
162 	else alias languageSeq = aliasSeqOf!(CTX.languages);
163 }
164 
165 /**
166 	Performs the string translation for a statically given language.
167 
168 	The second overload takes a plural form and a number to select from a set
169 	of translations based on the plural forms of the target language.
170 */
171 template tr(CTX, string LANG)
172 {
173 	string tr(string key, string context = null)
174 	{
175 		return tr!(CTX, LANG)(key, null, 0, context);
176 	}
177 
178 	string tr(string key, string key_plural, int n, string context = null)
179 	{
180 		static assert([languageSeq!CTX].canFind(LANG), "Unknown language: "~LANG);
181 
182 		foreach (i, mname; __traits(allMembers, CTX)) {
183 			static if (mname.startsWith(LANG~"_")) {
184 				enum langComponents = __traits(getMember, CTX, mname);
185 				foreach (entry; langComponents.messages) {
186 					if ((context is null) == (entry.context is null)) {
187 						if (context is null || entry.context == context) {
188 							if (entry.key == key) {
189 								if (key_plural !is null) {
190 									if (entry.pluralKey !is null && entry.pluralKey == key_plural) {
191 										static if (langComponents.nplurals_expr !is null && langComponents.plural_func_expr !is null) {
192 											mixin("int nplurals = "~langComponents.nplurals_expr~";");
193 											if (nplurals > 0) {
194 												mixin("int index = "~langComponents.plural_func_expr~";");
195 												return entry.pluralValues[index];
196 											}
197 											return entry.value;
198 										}
199 										assert(false, "Plural translations are not supported when the po file does not contain an entry for Plural-Forms.");
200 									}
201 								} else {
202 									return entry.value;
203 								}
204 							}
205 						}
206 					}
207 				}
208 			}
209 		}
210 
211 		static if (is(typeof(CTX.enforceExistingKeys)) && CTX.enforceExistingKeys) {
212 			if (key_plural !is null) {
213 				if (context is null) {
214 					assert(false, "Missing translation keys for "~LANG~": "~key~"&"~key_plural);
215 				}
216 				assert(false, "Missing translation key for "~LANG~"; "~context~": "~key~"&"~key_plural);
217 			}
218 
219 			if (context is null) {
220 				assert(false, "Missing translation key for "~LANG~": "~key);
221 			}
222 			assert(false, "Missing translation key for "~LANG~"; "~context~": "~key);
223 		} else {
224 			return n == 1 || !key_plural.length ? key : key_plural;
225 		}
226 	}
227 }
228 
229 /// Determines a language code from the value of a header string.
230 /// Returns: The best match from the Accept-Language header for a language. `null` if there is no supported language.
231 public string determineLanguageByHeader(T)(string accept_language, T allowed_languages) @safe pure @nogc
232 	if (isForwardRange!T && is(ElementType!T : string) || is(T == typeof(only())))
233 {
234 	import std.algorithm : splitter, countUntil;
235 	import std.string : indexOf;
236 
237 	// TODO: verify that allowed_languages doesn't contain a mix of languages with and without extra specifier for the same lanaguage (but only if one without specifier comes before those with specifier)
238 	// Implementing that feature should try to give a compile time warning and not change the behaviour of this function.
239 
240 	if (!accept_language.length)
241 		return null;
242 
243 	string fallback = null;
244 	foreach (accept; accept_language.splitter(",")) {
245 		auto sidx = accept.indexOf(';');
246 		if (sidx >= 0)
247 			accept = accept[0 .. sidx];
248 
249 		string alang, aextra;
250 		auto asep = accept.countUntil!(a => a == '_' || a == '-');
251 		if (asep < 0)
252 			alang = accept;
253 		else {
254 			alang = accept[0 .. asep];
255 			aextra = accept[asep + 1 .. $];
256 		}
257 
258 		static if (!is(T == typeof(only()))) { // workaround for type errors
259 			foreach (lang; allowed_languages.save) {
260 				string lcode, lextra;
261 				sidx = lang.countUntil!(a => a == '_' || a == '-');
262 				if (sidx < 0)
263 					lcode = lang;
264 				else {
265 					lcode = lang[0 .. sidx];
266 					lextra = lang[sidx + 1 .. $];
267 				}
268 				// request en_US == serve en_US
269 				if (lcode == alang && lextra == aextra)
270 					return lang;
271 				// request en_* == serve en
272 				if (lcode == alang && !lextra.length)
273 					return lang;
274 				// request en == serve en_*
275 				if (lcode == alang && !aextra.length)
276 					return lang;
277 				// request en* == serve en_* && be first occurence
278 				if (lcode == alang && lextra.length && !fallback.length)
279 					fallback = lang;
280 			}
281 		}
282 	}
283 
284 	return fallback;
285 }
286 
287 /// ditto
288 public string determineLanguageByHeader(Tuple...)(string accept_language, Tuple allowed_languages) @safe pure @nogc
289 	if (Tuple.length != 1 || is(Tuple[0] : string))
290 {
291 	return determineLanguageByHeader(accept_language, only(allowed_languages));
292 }
293 
294 /// ditto
295 public string determineLanguageByHeader(T)(HTTPServerRequest req, T allowed_languages) @safe pure
296 	if (isForwardRange!T && is(ElementType!T : string) || is(T == typeof(only())))
297 {
298 	return determineLanguageByHeader(req.headers.get("Accept-Language", null), allowed_languages);
299 }
300 
301 /// ditto
302 public string determineLanguageByHeader(Tuple...)(HTTPServerRequest req, Tuple allowed_languages) @safe pure
303 	if (Tuple.length != 1 || is(Tuple[0] : string))
304 {
305 	return determineLanguageByHeader(req.headers.get("Accept-Language", null), only(allowed_languages));
306 }
307 
308 @safe unittest {
309 	assert(determineLanguageByHeader("de-DE,de;q=0.8,en;q=0.6,en-US;q=0.4", ["en-US", "de_DE", "de_CH"]) == "de_DE");
310 	assert(determineLanguageByHeader("de-CH,de;q=0.8,en;q=0.6,en-US;q=0.4", ["en_US", "de_DE", "de-CH"]) == "de-CH");
311 	assert(determineLanguageByHeader("en_CA,en_US", ["ja_JP", "en"]) == "en");
312 	assert(determineLanguageByHeader("en", ["ja_JP", "en"]) == "en");
313 	assert(determineLanguageByHeader("en", ["ja_JP", "en_US"]) == "en_US");
314 	assert(determineLanguageByHeader("en_US", ["ja-JP", "en"]) == "en");
315 	assert(determineLanguageByHeader("de-DE,de;q=0.8,en;q=0.6,en-US;q=0.4", ["ja_JP"]) is null);
316 	assert(determineLanguageByHeader("de, de-DE ;q=0.8 , en ;q=0.6 , en-US;q=0.4", ["de-DE"]) == "de-DE");
317 	assert(determineLanguageByHeader("en_GB", ["en_US"]) == "en_US");
318 	assert(determineLanguageByHeader("de_DE", ["en_US"]) is null);
319 	assert(determineLanguageByHeader("en_US,enCA", ["en_GB"]) == "en_GB");
320 	assert(determineLanguageByHeader("en_US,enCA", ["en_GB", "en"]) == "en");
321 	assert(determineLanguageByHeader("en_US,enCA", ["en", "en_GB"]) == "en");
322 	assert(determineLanguageByHeader("de,en_US;q=0.8,en;q=0.6", ["en_US", "de_DE"]) == "de_DE");
323 	// TODO from above (should be invalid input having a more generic language first in the list!)
324 	//assert(determineLanguageByHeader("en_US,enCA", ["en", "en_US"]) == "en_US");
325 }
326 
327 package string determineLanguage(alias METHOD)(scope HTTPServerRequest req)
328 {
329 	alias CTX = GetTranslationContext!METHOD;
330 
331 	static if (!is(CTX == void)) {
332 		static if (is(typeof(CTX.determineLanguage(req)))) {
333 			static assert(is(typeof(CTX.determineLanguage(req)) == string),
334 				"determineLanguage in a translation context must return a language string.");
335 			return CTX.determineLanguage(req);
336 		} else {
337 			return determineLanguageByHeader(req, only(CTX.languages));
338 		}
339 	} else return null;
340 }
341 
342 unittest { // make sure that the custom determineLanguage is called
343 	static struct CTX {
344 		static string determineLanguage(HTTPServerRequest a) { return "test"; }
345 	}
346 	@translationContext!CTX
347 	static class Test {
348 		void test()
349 		{
350 		}
351 	}
352 	auto test = new Test;
353 	assert(determineLanguage!(test.test)(HTTPServerRequest.init) == "test");
354 }
355 
356 unittest { // issue #1955
357 	import std.meta : AliasSeq;
358 	import vibe.inet.url : URL;
359 	import vibe.http.server : createTestHTTPServerRequest;
360 
361 	static struct CTX {
362 		alias languages = AliasSeq!();
363 	}
364 
365 	@translationContext!CTX
366 	class C {
367 		void test() {}
368 	}
369 
370 	auto req = createTestHTTPServerRequest(URL("http://127.0.0.1/test"));
371 	assert(determineLanguage!(C.test)(req) == null);
372 }
373 
374 package template GetTranslationContext(alias METHOD)
375 {
376 	import vibe.internal.meta.uda;
377 
378 	alias PARENT = typeof(__traits(parent, METHOD).init);
379 	enum FUNCTRANS = findFirstUDA!(TranslationContextAttribute, METHOD);
380 	enum PARENTTRANS = findFirstUDA!(TranslationContextAttribute, PARENT);
381 	static if (FUNCTRANS.found) alias GetTranslationContext = FUNCTRANS.value.Context;
382 	else static if (PARENTTRANS.found) alias GetTranslationContext = PARENTTRANS.value.Context;
383 	else alias GetTranslationContext = void;
384 }
385 
386 
387 private struct DeclString {
388 	string context;
389 	string key;
390 	string pluralKey;
391 	string value;
392 	string[] pluralValues;
393 }
394 
395 private struct LangComponents {
396 	DeclString[] messages;
397 	string nplurals_expr;
398 	string plural_func_expr;
399 }
400 
401 // Example po header
402 /*
403  * # Translation of kstars.po into Spanish.
404  * # This file is distributed under the same license as the kdeedu package.
405  * # Pablo de Vicente <pablo@foo.com>, 2005, 2006, 2007, 2008.
406  * # Eloy Cuadra <eloy@bar.net>, 2007, 2008.
407  * msgid ""
408  * msgstr ""
409  * "Project-Id-Version: kstars\n"
410  * "Report-Msgid-Bugs-To: http://bugs.kde.org\n"
411  * "POT-Creation-Date: 2008-09-01 09:37+0200\n"
412  * "PO-Revision-Date: 2008-07-22 18:13+0200\n"
413  * "Last-Translator: Eloy Cuadra <eloy@bar.net>\n"
414  * "Language-Team: Spanish <kde-l10n-es@kde.org>\n"
415  * "MIME-Version: 1.0\n"
416  * "Content-Type: text/plain; charset=UTF-8\n"
417  * "Content-Transfer-Encoding: 8bit\n"
418  * "Plural-Forms: nplurals=2; plural=n != 1;\n"
419  */
420 
421 // PO format notes
422 /*
423  * # - Translator comment
424  * #: - source reference
425  * #. - extracted comments
426  * #, - flags such as "c-format" to indicate what king of substitutions may be present
427  * #| msgid - previous string comment
428  * #~ - obsolete message
429  * msgctxt - disabmbiguating context, like variable scope (optional, defaults to null)
430  * msgid - key to translate from (required)
431  * msgid_plural - plural form of the msg id (optional)
432  * msgstr - value to translate to (required)
433  * msgstr[0] - indexed translation for handling the various plural forms
434  * msgstr[1] - ditto
435  * msgstr[2] - ditto and etc...
436  */
437 
438 LangComponents extractDeclStrings(string text)
439 {
440 	DeclString[] declStrings;
441 	string nplurals_expr;
442 	string plural_func_expr;
443 
444 	size_t i = 0;
445 	while (true) {
446 		i = skipToDirective(i, text);
447 		if (i >= text.length) break;
448 
449 		string context = null;
450 
451 		// msgctxt is an optional field
452 		if (text.length - i >= 7 && text[i .. i+7] == "msgctxt") {
453 			i = skipWhitespace(i+7, text);
454 
455 			auto icntxt = skipString(i, text);
456 			context = dstringUnescape(wrapText(text[i+1 .. icntxt-1]));
457 			i = skipToDirective(icntxt, text);
458 		}
459 
460 		// msgid is a required field
461 		assert(text.length - i >= 5 && text[i .. i+5] == "msgid", "Expected 'msgid', got '"~text[i .. min(i+10, $)]~"'.");
462 		i += 5;
463 
464 		i = skipWhitespace(i, text);
465 
466 		auto iknext = skipString(i, text);
467 		auto key = dstringUnescape(wrapText(text[i+1 .. iknext-1]));
468 		i = iknext;
469 
470 		i = skipToDirective(i, text);
471 
472 		// msgid_plural is an optional field
473 		string key_plural = null;
474 		if (text.length - i >= 12 && text[i .. i+12] == "msgid_plural") {
475 			i = skipWhitespace(i+12, text);
476 			auto iprl = skipString(i, text);
477 			key_plural = dstringUnescape(wrapText(text[i+1 .. iprl-1]));
478 			i = skipToDirective(iprl, text);
479 		}
480 
481 		// msgstr is a required field
482 		assert(text.length - i >= 6 && text[i .. i+6] == "msgstr", "Expected 'msgstr', got '"~text[i .. min(i+10, $)]~"'.");
483 		i += 6;
484 
485 		string value;
486 		if (text[i] == '[') i -= 6;
487 		else {
488 			i = skipWhitespace(i, text);
489 			auto ivnext = skipString(i, text);
490 			value = dstringUnescape(wrapText(text[i+1 .. ivnext-1]));
491 			i = ivnext;
492 			i = skipToDirective(i, text);
493 		}
494 
495 		// msgstr[n] is a required field when msgid_plural is not null, and ignored otherwise
496 		string[] value_plural;
497 		if (key_plural !is null) {
498 			while (text.length - i >= 6 && text[i .. i+6] == "msgstr") {
499 				i = skipIndex(i+6, text);
500 				i = skipWhitespace(i, text);
501 				auto ims = skipString(i, text);
502 
503 				string plural = dstringUnescape(wrapText(text[i+1 .. ims-1]));
504 				i = skipLine(ims, text);
505 
506 				// Is it safe to assume that the entries are always sequential?
507 				value_plural ~= plural;
508 			}
509 		}
510 
511 		// Add the translation for the current language
512 		if (key == "") {
513 			nplurals_expr = parse_nplurals(value);
514 			plural_func_expr = parse_plural_expression(value);
515 		}
516 
517 		declStrings ~= DeclString(context, key, key_plural, value, value_plural);
518 	}
519 
520 	return LangComponents(declStrings, nplurals_expr, plural_func_expr);
521 }
522 
523 // Verify that two simple messages can be read and parsed correctly
524 unittest {
525 	auto str = `
526 # first string
527 msgid "ordinal.1"
528 msgstr "first"
529 
530 # second string
531 msgid "ordinal.2"
532 msgstr "second"`;
533 
534 	auto components = extractDeclStrings(str);
535 	auto ds = components.messages;
536 	assert(2 == ds.length, "Not enough DeclStrings have been processed");
537 	assert(ds[0].key == "ordinal.1", "The first key is not right.");
538 	assert(ds[0].value == "first", "The first value is not right.");
539 	assert(ds[1].key == "ordinal.2", "The second key is not right.");
540 	assert(ds[1].value == "second", "The second value is not right.");
541 }
542 
543 // Verify that the fields cannot be defined out of order
544 unittest {
545 	import core.exception : AssertError;
546 	import std.exception : assertThrown;
547 
548 	auto str1 = `
549 # unexpected field ahead
550 msgstr "world"
551 msgid "hello"`;
552 
553 	assertThrown!AssertError(extractDeclStrings(str1));
554 }
555 
556 // Verify that string wrapping is handled correctly
557 unittest {
558 	auto str = `
559 # The following text is wrapped
560 msgid ""
561 "This is an example of text that "
562 "has been wrapped on two lines."
563 msgstr ""
564 "It should not matter where it takes place, "
565 "the strings should all be concatenated properly."`;
566 
567 	auto ds = extractDeclStrings(str).messages;
568 	assert(1 == ds.length, "Expected one DeclString to have been processed.");
569 	assert(ds[0].key == "This is an example of text that has been wrapped on two lines.", "Failed to properly wrap the key");
570 	assert(ds[0].value == "It should not matter where it takes place, the strings should all be concatenated properly.", "Failed to properly wrap the key");
571 }
572 
573 // Verify that string wrapping and unescaping is handled correctly on example of PO headers
574 unittest {
575 	auto str = `
576 # English translations for ThermoWebUI package.
577 # This file is put in the public domain.
578 # Automatically generated, 2015.
579 #
580 msgid ""
581 msgstr ""
582 "Project-Id-Version: PROJECT VERSION\n"
583 "Report-Msgid-Bugs-To: developer@example.com\n"
584 "POT-Creation-Date: 2015-04-13 17:55+0600\n"
585 "PO-Revision-Date: 2015-04-13 14:13+0600\n"
586 "Last-Translator: Automatically generated\n"
587 "Language-Team: none\n"
588 "Language: en\n"
589 "MIME-Version: 1.0\n"
590 "Content-Type: text/plain; charset=UTF-8\n"
591 "Content-Transfer-Encoding: 8bit\n"
592 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
593 `;
594 	auto expected = `Project-Id-Version: PROJECT VERSION
595 Report-Msgid-Bugs-To: developer@example.com
596 POT-Creation-Date: 2015-04-13 17:55+0600
597 PO-Revision-Date: 2015-04-13 14:13+0600
598 Last-Translator: Automatically generated
599 Language-Team: none
600 Language: en
601 MIME-Version: 1.0
602 Content-Type: text/plain; charset=UTF-8
603 Content-Transfer-Encoding: 8bit
604 Plural-Forms: nplurals=2; plural=(n != 1);
605 `;
606 
607 	auto ds = extractDeclStrings(str).messages;
608 	assert(1 == ds.length, "Expected one DeclString to have been processed.");
609 	assert(ds[0].key == "", "Failed to properly wrap or unescape the key");
610 	assert(ds[0].value == expected, "Failed to properly wrap or unescape the value");
611 }
612 
613 // Verify that the message context is properly parsed
614 unittest {
615 	auto str1 = `
616 # "C" is for cookie
617 msgctxt "food"
618 msgid "C"
619 msgstr "C is for cookie, that's good enough for me."`;
620 
621 	auto ds1 = extractDeclStrings(str1).messages;
622 	assert(1 == ds1.length, "Expected one DeclString to have been processed.");
623 	assert(ds1[0].context == "food", "Expected a context of food");
624 	assert(ds1[0].key == "C", "Expected to find the letter C for the msgid.");
625 	assert(ds1[0].value == "C is for cookie, that's good enough for me.", "Unexpected value encountered for the msgstr.");
626 
627 	auto str2 = `
628 # No context validation
629 msgid "alpha"
630 msgstr "First greek letter."`;
631 
632 	auto ds2 = extractDeclStrings(str2).messages;
633 	assert(1 == ds2.length, "Expected one DeclString to have been processed.");
634 	assert(ds2[0].context is null, "Expected the context to be null when it is not defined.");
635 }
636 
637 unittest {
638 	enum str = `
639 # "C" is for cookie
640 msgctxt "food"
641 msgid "C"
642 msgstr "C is for cookie, that's good enough for me."
643 
644 # "C" is for language
645 msgctxt "lang"
646 msgid "C"
647 msgstr "Catalan"
648 
649 # Just "C"
650 msgid "C"
651 msgstr "Third letter"
652 `;
653 
654 	enum components = extractDeclStrings(str);
655 
656 	struct TranslationContext {
657 		import std.meta : AliasSeq;
658 		enum enforceExistingKeys = true;
659 		alias languages = AliasSeq!("en_US");
660 
661 		// Note that this is normally handled by mixing in an external file.
662 		enum en_US_unittest = components;
663 	}
664 
665 	auto newTr(string msgid, string msgcntxt = null) {
666 		return tr!(TranslationContext, "en_US")(msgid, msgcntxt);
667 	}
668 
669 	assert(newTr("C", "food") == "C is for cookie, that's good enough for me.", "Unexpected translation based on context.");
670 	assert(newTr("C", "lang") == "Catalan", "Unexpected translation based on context.");
671 	assert(newTr("C") == "Third letter", "Unexpected translation based on context.");
672 }
673 
674 unittest {
675 	enum str = `msgid ""
676 msgstr ""
677 "Project-Id-Version: kstars\\n"
678 "Plural-Forms: nplurals=2; plural=n != 1;\\n"
679 
680 msgid "One file was deleted."
681 msgid_plural "Files were deleted."
682 msgstr "One file was deleted."
683 msgstr[0] "1 file was deleted."
684 msgstr[1] "%d files were deleted."
685 
686 msgid "One file was created."
687 msgid_plural "Several files were created."
688 msgstr "One file was created."
689 msgstr[0] "1 file was created"
690 msgstr[1] "%d files were created."
691 
692 msgid "One file was modified."
693 msgid_plural "Several files were modified."
694 msgstr[0] "One file was modified."
695 msgstr[1] "%d files were modified."
696 `;
697 
698 	import std.stdio;
699 	enum components = extractDeclStrings(str);
700 
701 	struct TranslationContext {
702 		import std.meta : AliasSeq;
703 		enum enforceExistingKeys = true;
704 		alias languages = AliasSeq!("en_US");
705 
706 		// Note that this is normally handled by mixing in an external file.
707 		enum en_US_unittest2 = components;
708 	}
709 
710 	auto newTr(string msgid, string msgid_plural, int count, string msgcntxt = null) {
711 		return tr!(TranslationContext, "en_US")(msgid, msgid_plural, count, msgcntxt);
712 	}
713 
714 	string expected = "1 file was deleted.";
715 	auto actual = newTr("One file was deleted.", "Files were deleted.", 1);
716 	assert(expected == actual, "Expected: '"~expected~"' but got '"~actual~"'");
717 
718 	expected = "%d files were deleted.";
719 	actual = newTr("One file was deleted.", "Files were deleted.", 42);
720 	assert(expected == actual, "Expected: '"~expected~"' but got '"~actual~"'");
721 }
722 
723 private size_t skipToDirective(size_t i, ref string text)
724 {
725 	while (i < text.length) {
726 		i = skipWhitespace(i, text);
727 		if (i < text.length && text[i] == '#') i = skipLine(i, text);
728 		else break;
729 	}
730 	return i;
731 }
732 
733 private size_t skipWhitespace(size_t i, ref string text)
734 {
735 	while (i < text.length && (text[i] == ' ' || text[i] == '\t' || text[i] == '\n' || text[i] == '\r'))
736 		i++;
737 	return i;
738 }
739 
740 private size_t skipLine(size_t i, ref string text)
741 {
742 	while (i < text.length && text[i] != '\r' && text[i] != '\n') i++;
743 	if (i+1 < text.length && (text[i+1] == '\r' || text[i+1] == '\n') && text[i] != text[i+1]) i++;
744 	return i+1;
745 }
746 
747 private size_t skipString(size_t i, ref string text)
748 {
749 	import std.conv : to;
750 	assert(text[i] == '"', "Expected to encounter the start of a string at position: "~locationString(i, text));
751 	i++;
752 	while (true) {
753 		assert(i < text.length, "Missing closing '\"' for string: "~text[i .. min($, 10)]);
754 		if (text[i] == '"') {
755 			if (i+1 < text.length) {
756 				auto j = skipWhitespace(i+1, text);
757 				if (j<text.length && text[j] == '"') return skipString(j, text);
758 			}
759 			return i+1;
760 		}
761 		if (text[i] == '\\') i += 2;
762 		else i++;
763 	}
764 }
765 
766 private size_t skipIndex(size_t i, ref string text) {
767 	import std.conv : to;
768 	assert(text[i] == '[', "Expected to encounter a plural form of msgstr at position: "~locationString(i, text));
769 	for (; i<text.length; ++i) {
770 		if (text[i] == ']') {
771 			return i+1;
772 		}
773 	}
774 	assert(false, "Missing a ']' for a msgstr in a translation file.");
775 }
776 
777 private string wrapText(string str)
778 {
779 	string ret;
780 	bool wrapped = false;
781 
782 	for (size_t i=0; i<str.length; ++i) {
783 		if (str[i] == '\\') {
784 			assert(i+1 < str.length, "The string ends with the escape char: " ~ str);
785 			ret ~= str[i..i+2];
786 			++i;
787 		} else if (str[i] == '"') {
788 			wrapped = true;
789 			size_t j = skipWhitespace(i+1, str);
790 			if (j < str.length && str[j] == '"') {
791 				i=j;
792 			}
793 		} else ret ~= str[i];
794 	}
795 
796 	if (wrapped) return ret;
797 	return str;
798 }
799 
800 private string parse_nplurals(string msgstr)
801 in { assert(msgstr, "An empty string cannot be parsed for Plural-Forms."); }
802 do {
803 	import std.string : indexOf, CaseSensitive;
804 
805 	auto start = msgstr.indexOf("Plural-Forms:", CaseSensitive.no);
806 	if (start > -1) {
807 		auto beg = msgstr.indexOf("nplurals=", start+13, CaseSensitive.no);
808 		if (beg > -1) {
809 			auto end = msgstr.indexOf(';', beg+9, CaseSensitive.no);
810 			if (end > -1) {
811 				return msgstr[beg+9 .. end];
812 			}
813 			return msgstr[beg+9 .. $];
814 		}
815 	}
816 
817 	return null;
818 }
819 
820 unittest {
821 	auto res = parse_nplurals("Plural-Forms: nplurals=2; plural=n != 1;\n");
822 	assert(res == "2", "Failed to parse the correct number of plural forms for a language.");
823 }
824 
825 private string parse_plural_expression(string msgstr)
826 in { assert(msgstr, "An empty string cannot be parsed for Plural-Forms."); }
827 do {
828 	import std.string : indexOf, CaseSensitive;
829 
830 	auto start = msgstr.indexOf("Plural-Forms:", CaseSensitive.no);
831 	if (start > -1) {
832 		auto beg = msgstr.indexOf("plural=", start+13, CaseSensitive.no);
833 		if (beg > -1) {
834 			auto end = msgstr.indexOf(';', beg+7, CaseSensitive.no);
835 			if (end > -1) {
836 				return msgstr[beg+7 .. end];
837 			}
838 			return msgstr[beg+7 .. $];
839 		}
840 	}
841 
842 	return null;
843 }
844 
845 unittest {
846 	auto res = parse_plural_expression("Plural-Forms: nplurals=2; plural=n != 1;\n");
847 	assert(res == "n != 1", "Failed to parse the plural expression for a language.");
848 }
849 
850 private string dstringUnescape(in string str)
851 {
852 	string ret;
853 	size_t i, start = 0;
854 	for( i = 0; i < str.length; i++ )
855 		if( str[i] == '\\' ){
856 			if( i > start ){
857 				if( start > 0 ) ret ~= str[start .. i];
858 				else ret = str[0 .. i];
859 			}
860 			assert(i+1 < str.length, "The string ends with the escape char: " ~ str);
861 			switch(str[i+1]){
862 				default: ret ~= str[i+1]; break;
863 				case 'r': ret ~= '\r'; break;
864 				case 'n': ret ~= '\n'; break;
865 				case 't': ret ~= '\t'; break;
866 			}
867 			i++;
868 			start = i+1;
869 		}
870 
871 	if( i > start ){
872 		if( start == 0 ) return str;
873 		else ret ~= str[start .. i];
874 	}
875 	return ret;
876 }
877 
878 private string locationString(size_t i, string text)
879 {
880 	import std.algorithm.searching : count;
881 	import std.format : format;
882 	import std.string : lastIndexOf;
883 
884 	auto ln = text[0 .. i].count('\n');
885 	auto sep = text[0 .. i].lastIndexOf('\n');
886 	auto col = sep >= 0 ? i - sep - 1 : i;
887 	return format("line %s, column %s", ln+1, col+1);
888 }