1 /**
2 	MongoDB client connection settings.
3 
4 	Copyright: © 2012-2016 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig
7 */
8 module vibe.db.mongo.settings;
9 
10 import vibe.core.log;
11 import vibe.data.bson;
12 import vibe.db.mongo.flags : QueryFlags;
13 import vibe.inet.webform;
14 
15 import std.conv : to;
16 import std.digest : toHexString;
17 import std.digest.md : md5Of;
18 import std.algorithm : splitter, startsWith;
19 import std.string : icmp, indexOf, toLower;
20 
21 
22 /**
23  * Parses the given string as a mongodb URL. The URL must be in the form documented at
24  * $(LINK http://www.mongodb.org/display/DOCS/Connections) which is:
25  *
26  * mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]
27  *
28  * Returns: true if the URL was successfully parsed. False if the URL can not be parsed.
29  *
30  * If the URL is successfully parsed the MongoClientSettings instance will contain the parsed config.
31  * If the URL is not successfully parsed the information in the MongoClientSettings instance may be
32  * incomplete and should not be used.
33  */
34 bool parseMongoDBUrl(out MongoClientSettings cfg, string url)
35 @safe {
36 	import std.exception : enforce;
37 
38 	cfg = new MongoClientSettings();
39 
40 	string tmpUrl = url[0..$]; // Slice of the URL (not a copy)
41 
42 	if( !startsWith(tmpUrl, "mongodb://") )
43 	{
44 		return false;
45 	}
46 
47 	// Reslice to get rid of 'mongodb://'
48 	tmpUrl = tmpUrl[10..$];
49 
50 	auto authIndex = tmpUrl.indexOf('@');
51 	sizediff_t hostIndex = 0; // Start of the host portion of the URL.
52 
53 	// Parse out the username and optional password.
54 	if( authIndex != -1 )
55 	{
56 		// Set the host start to after the '@'
57 		hostIndex = authIndex + 1;
58 		string password;
59 
60 		auto colonIndex = tmpUrl[0..authIndex].indexOf(':');
61 		if(colonIndex != -1)
62 		{
63 			cfg.username = tmpUrl[0..colonIndex];
64 			password = tmpUrl[colonIndex + 1 .. authIndex];
65 		} else {
66 			cfg.username = tmpUrl[0..authIndex];
67 		}
68 
69 		// Make sure the username is not empty. If it is then the parse failed.
70 		if(cfg.username.length == 0)
71 		{
72 			return false;
73 		}
74 
75 		cfg.digest = MongoClientSettings.makeDigest(cfg.username, password);
76 	}
77 
78 	auto slashIndex = tmpUrl[hostIndex..$].indexOf("/");
79 	if( slashIndex == -1 ) slashIndex = tmpUrl.length;
80 	else slashIndex += hostIndex;
81 
82 	// Parse the hosts section.
83 	try
84 	{
85 		foreach(entry; splitter(tmpUrl[hostIndex..slashIndex], ","))
86 		{
87 			auto hostPort = splitter(entry, ":");
88 			string host = hostPort.front;
89 			hostPort.popFront();
90 			ushort port = MongoClientSettings.defaultPort;
91 			if (!hostPort.empty) {
92 				port = to!ushort(hostPort.front);
93 				hostPort.popFront();
94 			}
95 			enforce(hostPort.empty, "Host specifications are expected to be of the form \"HOST:PORT,HOST:PORT,...\".");
96 			cfg.hosts ~= MongoHost(host, port);
97 		}
98 	} catch (Exception e) {
99 		return  false; // Probably failed converting the port to ushort.
100 	}
101 
102 	// If we couldn't parse a host we failed.
103 	if(cfg.hosts.length == 0)
104 	{
105 		return false;
106 	}
107 
108 	if(slashIndex == tmpUrl.length)
109 	{
110 		// We're done parsing.
111 		return true;
112 	}
113 
114 	auto queryIndex = tmpUrl[slashIndex..$].indexOf("?");
115 	if(queryIndex == -1){
116 		// No query string. Remaining string is the database
117 		queryIndex = tmpUrl.length;
118 	} else {
119 		queryIndex += slashIndex;
120 	}
121 
122 	cfg.database = tmpUrl[slashIndex+1..queryIndex];
123 	if(queryIndex != tmpUrl.length)
124 	{
125 		FormFields options;
126 		parseURLEncodedForm(tmpUrl[queryIndex+1 .. $], options);
127 		foreach (option, value; options.byKeyValue) {
128 			bool setBool(ref bool dst)
129 			{
130 				try {
131 					dst = to!bool(value);
132 					return true;
133 				} catch( Exception e ){
134 					logError("Value for '%s' must be 'true' or 'false' but was '%s'.", option, value);
135 					return false;
136 				}
137 			}
138 
139 			bool setLong(ref long dst)
140 			{
141 				try {
142 					dst = to!long(value);
143 					return true;
144 				} catch( Exception e ){
145 					logError("Value for '%s' must be an integer but was '%s'.", option, value);
146 					return false;
147 				}
148 			}
149 
150 			void warnNotImplemented()
151 			{
152 				logDiagnostic("MongoDB option %s not yet implemented.", option);
153 			}
154 
155 			switch( option.toLower() ){
156 				default: logWarn("Unknown MongoDB option %s", option); break;
157 				case "appname": cfg.appName = value; break;
158 				case "slaveok": bool v; if( setBool(v) && v ) cfg.defQueryFlags |= QueryFlags.SlaveOk; break;
159 				case "replicaset": cfg.replicaSet = value; warnNotImplemented(); break;
160 				case "safe": setBool(cfg.safe); break;
161 				case "fsync": setBool(cfg.fsync); break;
162 				case "journal": setBool(cfg.journal); break;
163 				case "connecttimeoutms": setLong(cfg.connectTimeoutMS); warnNotImplemented(); break;
164 				case "sockettimeoutms": setLong(cfg.socketTimeoutMS); warnNotImplemented(); break;
165 				case "tls": setBool(cfg.ssl); break;
166 				case "ssl": setBool(cfg.ssl); break;
167 				case "sslverifycertificate": setBool(cfg.sslverifycertificate); break;
168 				case "authmechanism": cfg.authMechanism = parseAuthMechanism(value); break;
169 				case "wtimeoutms": setLong(cfg.wTimeoutMS); break;
170 				case "w":
171 					try {
172 						if(icmp(value, "majority") == 0){
173 							cfg.w = Bson("majority");
174 						} else {
175 							cfg.w = Bson(to!long(value));
176 						}
177 					} catch (Exception e) {
178 						logError("Invalid w value: [%s] Should be an integer number or 'majority'", value);
179 					}
180 				break;
181 			}
182 		}
183 
184 		/* Some m_settings imply safe. If they are set, set safe to true regardless
185 		 * of what it was set to in the URL string
186 		 */
187 		if( (cfg.w != Bson.init) || (cfg.wTimeoutMS != long.init) ||
188 				cfg.journal 	 || cfg.fsync )
189 		{
190 			cfg.safe = true;
191 		}
192 	}
193 
194 	return true;
195 }
196 
197 /* Test for parseMongoDBUrl */
198 unittest
199 {
200 	MongoClientSettings cfg;
201 
202 	assert(parseMongoDBUrl(cfg, "mongodb://localhost"));
203 	assert(cfg.hosts.length == 1);
204 	assert(cfg.database == "");
205 	assert(cfg.hosts[0].name == "localhost");
206 	assert(cfg.hosts[0].port == 27017);
207 	assert(cfg.defQueryFlags == QueryFlags.None);
208 	assert(cfg.replicaSet == "");
209 	assert(cfg.safe == false);
210 	assert(cfg.w == Bson.init);
211 	assert(cfg.wTimeoutMS == long.init);
212 	assert(cfg.fsync == false);
213 	assert(cfg.journal == false);
214 	assert(cfg.connectTimeoutMS == long.init);
215 	assert(cfg.socketTimeoutMS == long.init);
216 	assert(cfg.ssl == bool.init);
217 	assert(cfg.sslverifycertificate == true);
218 
219 	cfg = MongoClientSettings.init;
220 	assert(parseMongoDBUrl(cfg, "mongodb://fred:foobar@localhost"));
221 	assert(cfg.username == "fred");
222 	//assert(cfg.password == "foobar");
223 	assert(cfg.digest == MongoClientSettings.makeDigest("fred", "foobar"));
224 	assert(cfg.hosts.length == 1);
225 	assert(cfg.database == "");
226 	assert(cfg.hosts[0].name == "localhost");
227 	assert(cfg.hosts[0].port == 27017);
228 
229 	cfg = MongoClientSettings.init;
230 	assert(parseMongoDBUrl(cfg, "mongodb://fred:@localhost/baz"));
231 	assert(cfg.username == "fred");
232 	//assert(cfg.password == "");
233 	assert(cfg.digest == MongoClientSettings.makeDigest("fred", ""));
234 	assert(cfg.database == "baz");
235 	assert(cfg.hosts.length == 1);
236 	assert(cfg.hosts[0].name == "localhost");
237 	assert(cfg.hosts[0].port == 27017);
238 
239 	cfg = MongoClientSettings.init;
240 	assert(parseMongoDBUrl(cfg, "mongodb://host1,host2,host3/?safe=true&w=2&wtimeoutMS=2000&slaveOk=true&ssl=true&sslverifycertificate=false"));
241 	assert(cfg.username == "");
242 	//assert(cfg.password == "");
243 	assert(cfg.digest == "");
244 	assert(cfg.database == "");
245 	assert(cfg.hosts.length == 3);
246 	assert(cfg.hosts[0].name == "host1");
247 	assert(cfg.hosts[0].port == 27017);
248 	assert(cfg.hosts[1].name == "host2");
249 	assert(cfg.hosts[1].port == 27017);
250 	assert(cfg.hosts[2].name == "host3");
251 	assert(cfg.hosts[2].port == 27017);
252 	assert(cfg.safe == true);
253 	assert(cfg.w == Bson(2L));
254 	assert(cfg.wTimeoutMS == 2000);
255 	assert(cfg.defQueryFlags == QueryFlags.SlaveOk);
256 	assert(cfg.ssl == true);
257 	assert(cfg.sslverifycertificate == false);
258 
259 	cfg = MongoClientSettings.init;
260 	assert(parseMongoDBUrl(cfg,
261 				"mongodb://fred:flinstone@host1.example.com,host2.other.example.com:27108,host3:"
262 				~ "27019/mydb?journal=true;fsync=true;connectTimeoutms=1500;sockettimeoutMs=1000;w=majority"));
263 	assert(cfg.username == "fred");
264 	//assert(cfg.password == "flinstone");
265 	assert(cfg.digest == MongoClientSettings.makeDigest("fred", "flinstone"));
266 	assert(cfg.database == "mydb");
267 	assert(cfg.hosts.length == 3);
268 	assert(cfg.hosts[0].name == "host1.example.com");
269 	assert(cfg.hosts[0].port == 27017);
270 	assert(cfg.hosts[1].name == "host2.other.example.com");
271 	assert(cfg.hosts[1].port == 27108);
272 	assert(cfg.hosts[2].name == "host3");
273 	assert(cfg.hosts[2].port == 27019);
274 	assert(cfg.fsync == true);
275 	assert(cfg.journal == true);
276 	assert(cfg.connectTimeoutMS == 1500);
277 	assert(cfg.socketTimeoutMS == 1000);
278 	assert(cfg.w == Bson("majority"));
279 	assert(cfg.safe == true);
280 
281 	// Invalid URLs - these should fail to parse
282 	cfg = MongoClientSettings.init;
283 	assert(! (parseMongoDBUrl(cfg, "localhost:27018")));
284 	assert(! (parseMongoDBUrl(cfg, "http://blah")));
285 	assert(! (parseMongoDBUrl(cfg, "mongodb://@localhost")));
286 	assert(! (parseMongoDBUrl(cfg, "mongodb://:thepass@localhost")));
287 	assert(! (parseMongoDBUrl(cfg, "mongodb://:badport/")));
288 
289 	assert(parseMongoDBUrl(cfg, "mongodb://me:sl$ash/w0+rd@localhost"));
290 	assert(cfg.digest == MongoClientSettings.makeDigest("me", "sl$ash/w0+rd"));
291 	assert(cfg.hosts.length == 1);
292 	assert(cfg.hosts[0].name == "localhost");
293 	assert(cfg.hosts[0].port == 27017);
294 	assert(parseMongoDBUrl(cfg, "mongodb://me:sl$ash/w0+rd@localhost/mydb"));
295 	assert(cfg.digest == MongoClientSettings.makeDigest("me", "sl$ash/w0+rd"));
296 	assert(cfg.database == "mydb");
297 	assert(cfg.hosts.length == 1);
298 	assert(cfg.hosts[0].name == "localhost");
299 	assert(cfg.hosts[0].port == 27017);
300 }
301 
302 /**
303  * Describes a vibe.d supported authentication mechanism to use on client
304  * connection to a MongoDB server.
305  */
306 enum MongoAuthMechanism
307 {
308 	/**
309 	 * Use no auth mechanism. If a digest or ssl certificate is given this
310 	 * defaults to trying the recommend auth mechanisms depending on server
311 	 * version and input parameters.
312 	 */
313 	none,
314 
315 	/**
316 	 * Use SCRAM-SHA-1 as defined in [RFC 5802](http://tools.ietf.org/html/rfc5802)
317 	 *
318 	 * This is the default when a password is provided. In the future other
319 	 * scram algorithms may be implemented and selectable through these values.
320 	 *
321 	 * MongoDB: 3.0–
322 	 */
323 	scramSHA1,
324 
325 	/**
326 	 * Forces login through the legacy MONGODB-CR authentication mechanism. This
327 	 * mechanism is a nonce and MD5 based system.
328 	 *
329 	 * MongoDB: 1.4–4.0 (deprecated 3.0)
330 	 */
331 	mongoDBCR,
332 
333 	/**
334 	 * Use an X.509 certificate to authenticate. Only works if digest is set to
335 	 * null or empty string in the MongoClientSettings.
336 	 *
337 	 * MongoDB: 2.6–
338 	 */
339 	mongoDBX509
340 }
341 
342 private MongoAuthMechanism parseAuthMechanism(string str)
343 @safe {
344 	switch (str) {
345 		case "SCRAM-SHA-1": return MongoAuthMechanism.scramSHA1;
346 		case "MONGODB-CR": return MongoAuthMechanism.mongoDBCR;
347 		case "MONGODB-X509": return MongoAuthMechanism.mongoDBX509;
348 		default: throw new Exception("Auth mechanism \"" ~ str ~ "\" not supported");
349 	}
350 }
351 
352 /**
353  * See_Also: $(LINK https://docs.mongodb.com/manual/reference/connection-string/#connections-connection-options)
354  */
355 class MongoClientSettings
356 {
357 	/// Gets the default port used for MongoDB connections
358 	enum ushort defaultPort = 27017;
359 
360 	/**
361 	 * If set to non-empty string, use this username to try to authenticate with
362 	 * to the database. Only has an effect if digest or sslPEMKeyFile is set too
363 	 *
364 	 * Use $(LREF authenticatePassword) or $(LREF authenticateSSL) to
365 	 * automatically fill this.
366 	 */
367 	string username;
368 
369 	/**
370 	 * The password hashed as MongoDB digest as returned by $(LREF makeDigest).
371 	 *
372 	 * **DISCOURAGED** to fill this manually as future authentication mechanisms
373 	 * may use other digest algorithms.
374 	 *
375 	 * Use $(LREF authenticatePassword) to automatically fill this.
376 	 */
377 	string digest;
378 
379 	/**
380 	 * Amount of maximum simultaneous connections to have open at the same time.
381 	 *
382 	 * Every MongoDB call may allocate a new connection if no previous ones are
383 	 * available and there is no connection associated with the calling Fiber.
384 	 */
385 	uint maxConnections = uint.max;
386 
387 	/**
388 	 * MongoDB hosts to try to connect to.
389 	 *
390 	 * Bugs: currently only a connection to the first host is attempted, more
391 	 * hosts are simply ignored.
392 	 */
393 	MongoHost[] hosts;
394 
395 	/**
396 	 * Default auth database to operate on, otherwise operating on special
397 	 * "admin" database for all MongoDB authentication commands.
398 	 */
399 	string database;
400 
401 	/**
402 	 * Flags to use on all database query commands. The
403 	 * $(REF slaveOk, vibe,db,mongo,flags,QueryFlags) bit may be set using the
404 	 * "slaveok" query parameter inside the MongoDB URL.
405 	 */
406 	QueryFlags defQueryFlags = QueryFlags.None;
407 
408 	/**
409 	 * Specifies the name of the replica set, if the mongod is a member of a
410 	 * replica set.
411 	 *
412 	 * Bugs: Not yet implemented
413 	 */
414 	string replicaSet;
415 
416 	/**
417 	 * Automatically check for errors when operating on collections and throw a
418 	 * $(REF MongoDBException, vibe,db,mongo,connection) in case of errors.
419 	 *
420 	 * Automatically set if either:
421 	 * * the "w" (write concern) parameter is set
422 	 * * the "wTimeoutMS" parameter is set
423 	 * * journal is true
424 	 */
425 	bool safe;
426 
427 	/**
428 	 * Requests acknowledgment that write operations have propagated to a
429 	 * specified number of mongod instances (number) or to mongod instances with
430 	 * specified tags (string) or "majority" for calculated majority.
431 	 *
432 	 * See_Also: write concern [w Option](https://docs.mongodb.com/manual/reference/write-concern/#wc-w).
433 	 */
434 	Bson w; // Either a number or the string 'majority'
435 
436 	/**
437 	 * Time limit for the w option to prevent write operations from blocking
438 	 * indefinitely.
439 	 *
440 	 * See_Also: $(LREF w)
441 	 */
442 	long wTimeoutMS;
443 
444 	// undocumented feature in no documentation of >=MongoDB 2.2 ?!
445 	bool fsync;
446 
447 	/**
448 	 * Requests acknowledgment that write operations have been written to the
449 	 * [on-disk journal](https://docs.mongodb.com/manual/core/journaling/).
450 	 *
451 	 * See_Also: write concern [j Option](https://docs.mongodb.com/manual/reference/write-concern/#wc-j).
452 	 */
453 	bool journal;
454 
455 	/**
456 	 * The time in milliseconds to attempt a connection before timing out.
457 	 *
458 	 * Bugs: Not yet implemented
459 	 */
460 	long connectTimeoutMS;
461 
462 	/**
463 	 * The time in milliseconds to attempt a send or receive on a socket before
464 	 * the attempt times out.
465 	 *
466 	 * Bugs: Not yet implemented
467 	 */
468 	long socketTimeoutMS;
469 
470 	/**
471 	 * Enables or disables TLS/SSL for the connection.
472 	 */
473 	bool ssl;
474 
475 	/**
476 	 * Can be set to false to disable TLS peer validation to allow self signed
477 	 * certificates.
478 	 *
479 	 * This mode is discouraged and should ONLY be used in development.
480 	 */
481 	bool sslverifycertificate = true;
482 
483 	/**
484 	 * Path to a certificate with private key and certificate chain to connect
485 	 * with.
486 	 */
487 	string sslPEMKeyFile;
488 
489 	/**
490 	 * Path to a certificate authority file for verifying the remote
491 	 * certificate.
492 	 */
493 	string sslCAFile;
494 
495 	/**
496 	 * Use the given authentication mechanism when connecting to the server. If
497 	 * unsupported by the server, throw a MongoAuthException.
498 	 *
499 	 * If set to none, but digest or sslPEMKeyFile are set, this automatically
500 	 * determines a suitable authentication mechanism based on server version.
501 	 */
502 	MongoAuthMechanism authMechanism;
503 
504 	/**
505 	 * Application name for the connection information when connected.
506 	 *
507 	 * The application name is printed to the mongod logs upon establishing the
508 	 * connection. It is also recorded in the slow query logs and profile
509 	 * collections.
510 	 */
511 	string appName;
512 
513 	/**
514 	 * Generates a digest string which can be used for authentication by setting
515 	 * the username and digest members.
516 	 *
517 	 * Use $(LREF authenticate) to automatically configure username and digest.
518 	 */
519 	static pure string makeDigest(string username, string password)
520 	@safe {
521 		return md5Of(username ~ ":mongo:" ~ password).toHexString().idup.toLower();
522 	}
523 
524 	/**
525 	 * Sets the username and the digest string in this MongoClientSettings
526 	 * instance.
527 	 */
528 	void authenticatePassword(string username, string password)
529 	@safe {
530 		this.username = username;
531 		this.digest = MongoClientSettings.makeDigest(username, password);
532 	}
533 
534 	/**
535 	 * Sets ssl, the username, the PEM key file and the trusted CA file in this
536 	 * MongoClientSettings instance.
537 	 *
538 	 * Params:
539 	 *   username = The username as provided in the cert file like
540 	 *   `"C=IS,ST=Reykjavik,L=Reykjavik,O=MongoDB,OU=Drivers,CN=client"`.
541 	 *
542 	 *   The username can be blank if connecting to MongoDB 3.4 or above.
543 	 *
544 	 *   sslPEMKeyFile = Path to a certificate with private key and certificate
545 	 *   chain to connect with.
546 	 *
547 	 *   sslCAFile = Optional path to a trusted certificate authority file for
548 	 *   verifying the remote certificate.
549 	 */
550 	void authenticateSSL(string username, string sslPEMKeyFile, string sslCAFile = null)
551 	@safe {
552 		this.ssl = true;
553 		this.digest = null;
554 		this.username = username;
555 		this.sslPEMKeyFile = sslPEMKeyFile;
556 		this.sslCAFile = sslCAFile;
557 	}
558 }
559 
560 /// Describes a host we might be able to connect to
561 struct MongoHost
562 {
563 	/// The host name or IP address of the remote MongoDB server.
564 	string name;
565 	/// The port of the MongoDB server. See `MongoClientSettings.defaultPort`.
566 	ushort port;
567 }