1 /**
2 	Type safe implementations of common Redis storage idioms.
3 
4 	Note that the API is still subject to change!
5 
6 	Copyright: © 2014 RejectedSoftware e.K.
7 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
8 	Authors: Sönke Ludwig
9 */
10 module vibe.db.redis.idioms;
11 
12 import vibe.db.redis.redis;
13 import vibe.db.redis.types;
14 
15 import core.time : msecs, seconds;
16 
17 
18 /**
19 */
20 struct RedisCollection(T /*: RedisValue*/, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
21 {
22 	static assert(ID_LENGTH > 0, "IDs must have a length of at least one.");
23 	static assert(!(OPTIONS & RedisCollectionOptions.supportIteration) || ID_LENGTH == 1, "ID generation currently not supported for ID lengths greater 2.");
24 
25 	alias IDS = Replicate!(long, ID_LENGTH);
26 	static if (ID_LENGTH == 1) alias IDType = long;
27 	else alias IDType = Tuple!IDS;
28 
29 	private {
30 		RedisDatabase m_db;
31 		string[ID_LENGTH] m_prefix;
32 		string m_suffix;
33 		static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) {
34 			@property string m_idCounter() const { return m_prefix[0] ~ "max"; }
35 			@property string m_allSet() const { return m_prefix[0] ~ "all"; }
36 		}
37 	}
38 
39 	this(RedisDatabase db, Replicate!(string, ID_LENGTH) name, string suffix = null)
40 	{
41 		initialize(db, name, suffix);
42 	}
43 
44 	void initialize(RedisDatabase db, Replicate!(string, ID_LENGTH) name, string suffix = null)
45 	{
46 		m_db = db;
47 		foreach (i, N; name) {
48 			if (i == 0) m_prefix[i] = name[i] ~ ":";
49 			else m_prefix[i] = ":" ~ name[i] ~ ":";
50 		}
51 		if (suffix.length) m_suffix = ":" ~ suffix;
52 	}
53 
54 	@property inout(RedisDatabase) database() inout { return m_db; }
55 
56 	T opIndex(IDS id) { return T(m_db, getKey(id)); }
57 
58 	static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) {
59 		/** Creates an ID without setting a corresponding value.
60 		*/
61 		IDType createID()
62 		{
63 			auto id = m_db.incr(m_idCounter);
64 			static if (OPTIONS & RedisCollectionOptions.supportPaging)
65 				m_db.zadd(m_allSet, id, id);
66 			else m_db.sadd(m_allSet, id);
67 			return id;
68 		}
69 
70 		IDType add(U)(U args)
71 		{
72 			auto id = createID();
73 			this[id] = args;
74 			return id;
75 		}
76 
77 		bool isMember(long id)
78 		{
79 			static if (OPTIONS & RedisCollectionOptions.supportPaging)
80 				return m_db.zscore(m_allSet, id).hasNext();
81 			else return m_db.sisMember(m_allSet, id);
82 		}
83 
84 		static if (OPTIONS & RedisCollectionOptions.supportPaging) {
85 			// TODO: add range queries
86 		}
87 
88 		int opApply(int delegate(long id) del)
89 		{
90 			static if (OPTIONS & RedisCollectionOptions.supportPaging) {
91 				foreach (id; m_db.zrange!long(m_allSet, 0, -1))
92 					if (auto ret = del(id))
93 						return ret;
94 			} else {
95 				foreach (id; m_db.smembers!long(m_allSet))
96 					if (auto ret = del(id))
97 						return ret;
98 			}
99 			return 0;
100 		}
101 
102 		int opApply(int delegate(long id, T) del)
103 		{
104 			static if (OPTIONS & RedisCollectionOptions.supportPaging) {
105 				foreach (id; m_db.zrange!long(m_allSet, 0, -1))
106 					if (auto ret = del(id, this[id]))
107 						return ret;
108 			} else {
109 				foreach (id; m_db.smembers!long(m_allSet))
110 					if (auto ret = del(id, this[id]))
111 						return ret;
112 			}
113 			return 0;
114 		}
115 	}
116 
117 	/** Removes an ID along with the corresponding value.
118 	*/
119 	void remove(IDS id)
120 	{
121 		this[id].remove();
122 		static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) {
123 			static if (OPTIONS & RedisCollectionOptions.supportPaging)
124 				m_db.zrem(m_allSet, id);
125 			else
126 				m_db.srem(m_allSet, id);
127 		}
128 	}
129 
130 
131 	private string getKey(IDS ids)
132 	{
133 		import std.conv;
134 		static if (ID_LENGTH == 1) {
135 			return m_prefix[0] ~ ids.to!string ~ m_suffix;
136 		} else {
137 			string ret;
138 			foreach (i, id; ids) ret ~= m_prefix[i] ~ id.to!string;
139 			return ret ~ m_suffix;
140 		}
141 	}
142 }
143 
144 enum RedisCollectionOptions {
145 	none             = 0,    // Plain collection without iteration/paging support
146 	supportIteration = 1<<0, // Store IDs in a set to be able to iterate and check for existence
147 	supportPaging    = 1<<1, // Store IDs in a sorted set, to support range based queries
148 	defaults = supportIteration
149 }
150 
151 
152 /** Models a set of numbered hashes.
153 
154 	This structure is roughly equivalent to a $(D string[string][long]) and is
155 	commonly used to store collections of objects, such as all users of a
156 	service. For a strongly typed variant of this class, see
157 	$(D RedisObjectCollection).
158 
159 	See_also: $(D RedisObjectCollection)
160 */
161 template RedisHashCollection(RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
162 {
163 	alias RedisHashCollection = RedisCollection!(RedisHash, OPTIONS, ID_LENGTH);
164 }
165 
166 
167 /** Models a strongly typed set of numbered hashes.
168 
169 	This structure is roughly equivalent of a $(D T[long]).
170 
171 	See_also: $(D RedisHashCollection)
172 */
173 template RedisObjectCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
174 {
175 	alias RedisObjectCollection = RedisCollection!(RedisObject!T, OPTIONS, ID_LENGTH);
176 }
177 
178 ///
179 unittest {
180 	struct User {
181 		string name;
182 		string email;
183 		int age;
184 		string password;
185 	}
186 
187 	void test()
188 	{
189 		auto db = connectRedis("127.0.0.1").getDatabase(0);
190 		db.deleteAll();
191 
192 		auto users = RedisObjectCollection!User(db, "users");
193 		assert(users.add(User("Tom", "tom@example.com", 42, "secret")) == 0);
194 		assert(users.add(User("Peter", "peter@example.com", 42, "secret")) == 1);
195 
196 		auto peter = users[1];
197 		assert(peter.name == "Peter");
198 	}
199 }
200 
201 
202 /** Models a single strongly typed object.
203 
204 	This structure is rougly equivalent to a value of type $(D T). The
205 	underlying data is represented as a Redis hash. This means that only
206 	primitive fields are supported for $(D T).
207 */
208 struct RedisObject(T) {
209 	private {
210 		RedisHash!string m_hash;
211 	}
212 
213 	this(RedisDatabase db, string key)
214 	{
215 		m_hash = RedisHash!string(db, key);
216 	}
217 
218 	this(RedisHash!string hash)
219 	{
220 		m_hash = hash;
221 	}
222 
223 	@property T get()
224 	{
225 		T ret;
226 		auto repl = m_hash.database.hmget(m_hash.key, keys);
227 		foreach (i, F; typeof(ret.tupleof)) {
228 			assert(!repl.empty);
229 			__traits(getMember, ret, keys[i]) = repl.front.fromRedis!F;
230 			repl.popFront();
231 		}
232 		assert(repl.empty);
233 		return ret;
234 	}
235 
236 	@property bool exists() { return m_hash.value.exists(); }
237 
238 	alias get this;
239 
240 	void remove() { m_hash.remove(); }
241 
242 	void opAssign(T val)
243 	{
244 		m_hash.database.hmset(m_hash.key, toTuple(toKeysAndValues(val)).expand);
245 	}
246 
247 	mixin(fields());
248 
249 	static private string fields()
250 	{
251 		string ret;
252 		foreach (name; keys) {
253 			ret ~= "@property auto "~name~"() { return RedisObjectField!(typeof(T."~name~"))(m_hash, \""~name~"\"); }\n";
254 			ret ~= "@property void "~name~"(typeof(T."~name~") val) { this."~name~".opAssign(val); }\n";
255 		}
256 		return ret;
257 	}
258 
259 	/*@property auto opDispatch(string name)() //if (is(typeof(getMember, T.init, name)))
260 	{
261 		return RedisObjectField!(typeof(__traits(getMember, T.init, name)))(m_hash, name);
262 	}*/
263 
264 	private static string[T.tupleof.length*2] toKeysAndValues(T val)
265 	{
266 		string[T.tupleof.length*2] ret;
267 		enum keys = fieldNames!T;
268 		foreach (i, m; val.tupleof) {
269 			ret[i*2+0] = keys[i];
270 			ret[i*2+1] = m.toRedis();
271 		}
272 		return ret;
273 	}
274 
275 	private enum keys = fieldNames!T;
276 }
277 
278 struct RedisObjectField(T) {
279 	private {
280 		RedisHash!string m_hash;
281 		string m_field;
282 	}
283 
284 	this(RedisHash!string hash, string field)
285 	{
286 		m_hash = hash;
287 		m_field = field;
288 	}
289 
290 	@property T get() { return m_hash.database.hget!string(m_hash.key, m_field).fromRedis!T; }
291 
292 	alias get this;
293 
294 	void opAssign(T val) { m_hash.database.hset(m_hash.key, m_field, val.toRedis); }
295 
296 	void opUnary(string op)() if(op == "++") { m_hash.database.hincr(m_hash.key, m_field, 1); }
297 	void opUnary(string op)() if(op == "--") { m_hash.database.hincr(m_hash.key, m_field, -1); }
298 
299 	void opOpAssign(string op)(long val) if (op == "+") { m_hash.database.hincr(m_hash.key, m_field, val); }
300 	void opOpAssign(string op)(long val) if (op == "-") { m_hash.database.hincr(m_hash.key, m_field, -val); }
301 	void opOpAssign(string op)(double val) if (op == "+") { m_hash.database.hincr(m_hash.key, m_field, val); }
302 	void opOpAssign(string op)(double val) if (op == "-") { m_hash.database.hincr(m_hash.key, m_field, -val); }
303 }
304 
305 
306 /** Models a strongly typed numbered set of values.
307 
308 
309 */
310 template RedisSetCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
311 {
312 	alias RedisSetCollection = RedisCollection!(RedisSet!T, OPTIONS, ID_LENGTH);
313 }
314 
315 ///
316 unittest {
317 	void test()
318 	{
319 		auto db = connectRedis("127.0.0.1").getDatabase(0);
320 		auto user_groups = RedisSetCollection!(string, RedisCollectionOptions.none)(db, "user_groups");
321 
322 		// add some groups for user with ID 0
323 		user_groups[0].insert("cooking");
324 		user_groups[0].insert("hiking");
325 		// add some groups for user with ID 1
326 		user_groups[1].insert("coding");
327 
328 		assert(user_groups[0].contains("hiking"));
329 		assert(!user_groups[0].contains("coding"));
330 		assert(user_groups[1].contains("coding"));
331 
332 		user_groups[0].remove("hiking");
333 		assert(!user_groups[0].contains("hiking"));
334 	}
335 }
336 
337 
338 /** Models a strongly typed numbered set of values.
339 
340 
341 */
342 template RedisListCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
343 {
344 	alias RedisListCollection = RedisCollection!(RedisList!T, OPTIONS, ID_LENGTH);
345 }
346 
347 
348 /** Models a strongly typed numbered set of values.
349 
350 
351 */
352 template RedisStringCollection(T = string, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
353 {
354 	alias RedisStringCollection = RedisCollection!(RedisString!T, OPTIONS, ID_LENGTH);
355 }
356 
357 
358 // TODO: support distributed locking
359 struct RedisLock {
360 	private {
361 		RedisDatabase m_db;
362 		string m_key;
363 		string m_scriptSHA;
364 	}
365 
366 	this(RedisDatabase db, string lock_key)
367 	{
368 		m_db = db;
369 		m_key = lock_key;
370 		m_scriptSHA = m_db.scriptLoad(
371 `if redis.call("get",KEYS[1]) == ARGV[1] then
372 	return redis.call("del",KEYS[1])
373 else
374 	return 0
375 end`);
376 	}
377 
378 	void performLocked(scope void delegate() del)
379 	{
380 		import std.random;
381 		import vibe.core.core;
382 		import vibe.data.bson;
383 
384 		auto lockval = BsonObjectID.generate();
385 		while (!m_db.setNX(m_key, cast(ubyte[])lockval, 30.seconds))
386 			sleep(uniform(1, 50).msecs);
387 
388 		scope (exit) m_db.evalSHA!(string, ubyte[])(m_scriptSHA, [m_key], cast(ubyte[])lockval);
389 
390 		del();
391 	}
392 }
393 
394 
395 // utility structure, temporarily placed here
396 struct JsonEncoded(T) {
397 	import vibe.data.json;
398 	T value;
399 
400 	alias value this;
401 
402 	static JsonEncoded fromString(string str) { return JsonEncoded(deserializeJson!T(str)); }
403 	string toString() { return serializeToJsonString(value); }
404 
405 	static assert(isStringSerializable!JsonEncoded);
406 }
407 JsonEncoded!T jsonEncoded(T)(T value) { return JsonEncoded!T(value); }
408 
409 
410 // utility structure, temporarily placed here
411 struct LazyString(T...) {
412 	private {
413 		T m_values;
414 	}
415 
416 	this(T values) { m_values = values; }
417 
418 	void toString(void delegate(string) sink)
419 	{
420 		foreach (v; m_values)
421 			dst.formattedWrite("%s", v);
422 	}
423 }
424 
425 
426 /**
427 	Strips all non-Redis fields from a struct.
428 
429 	The returned struct will contain only fiels that can be converted using
430 	$(D toRedis) and that have names different than "id" or "_id".
431 
432 	To reconstruct the full struct type, use the $(D RedisStripped.unstrip)
433 	method.
434 */
435 RedisStripped!(T, strip_id) redisStrip(bool strip_id = true, T)(in T val) { return RedisStripped!(T, strip_id)(val); }
436 
437 /**
438 	Represents the stripped type of a struct.
439 
440 	Strips all fields that cannot be directly stored as values in the Redis
441 	database. By default, any field named `id` or `_id` is also stripped. Set
442 	the `strip_id` parameter to `false` to keep those fields.
443 
444 	See_also: $(D redisStrip)
445 */
446 struct RedisStripped(T, bool strip_id = true) {
447 	import std.traits : Select, select;
448 	import std.typetuple;
449 
450 	//pragma(msg, membersString!());
451 	mixin(membersString());
452 
453 	alias StrippedMembers = FilterToType!(Select!(strip_id, isNonRedisTypeOrID, isNonRedisType), T.tupleof);
454 	alias UnstrippedMembers = FilterToType!(Select!(strip_id, isRedisTypeAndNotID, isRedisType), T.tupleof);
455 	alias strippedMemberIndices = indicesOf!(Select!(strip_id, isNonRedisTypeOrID, isNonRedisType), T.tupleof);
456 	alias unstrippedMemberIndices = indicesOf!(Select!(strip_id, isRedisTypeAndNotID, isRedisType), T.tupleof);
457 
458 	this(in T src) { foreach (i, idx; unstrippedMemberIndices) this.tupleof[i] = src.tupleof[idx]; }
459 
460 	/** Reconstructs the full (unstripped) struct value.
461 
462 		The parameters for this method are all stripped fields in the order in
463 		which they appear in the original struct definition.
464 	*/
465 	T unstrip(StrippedMembers stripped_members) {
466 		T ret;
467 		populateRedisFields(ret, this.tupleof);
468 		populateNonRedisFields(ret, stripped_members);
469 		return ret;
470 	}
471 
472 	private void populateRedisFields(ref T dst, UnstrippedMembers values)
473 	{
474 		foreach (i, v; values)
475 			dst.tupleof[unstrippedMemberIndices[i]] = v;
476 	}
477 
478 	private void populateNonRedisFields(ref T dst, StrippedMembers values)
479 	{
480 		foreach (i, v; values)
481 			dst.tupleof[strippedMemberIndices[i]] = v;
482 	}
483 
484 
485 	/*pragma(msg, T);
486 	pragma(msg, "stripped: "~StrippedMembers.stringof~" - "~strippedMemberIndices.stringof);
487 	pragma(msg, "unstripped: "~UnstrippedMembers.stringof~" - "~unstrippedMemberIndices.stringof);*/
488 
489 	private static string membersString()
490 	{
491 		string ret;
492 		foreach (idx; unstrippedMemberIndices) {
493 			enum name = __traits(identifier, T.tupleof[idx]);
494 			ret ~= "typeof(T."~name~") "~name~";\n";
495 		}
496 		return ret;
497 	}
498 }
499 
500 unittest {
501 	static struct S1 { int id; string field; string[] array; }
502 	auto s1 = S1(42, "hello", ["world"]);
503 	auto s1s = redisStrip(s1);
504 	static assert(!is(typeof(s1s.id)));
505 	static assert(is(typeof(s1s.field)));
506 	static assert(!is(typeof(s1s.array)));
507 	assert(s1s.field == "hello");
508 	auto s1u = s1s.unstrip(42, ["world"]);
509 	assert(s1u == s1);
510 }
511 
512 private template indicesOf(alias PRED, T...)
513 {
514 	import std.typetuple;
515 	template impl(size_t i) {
516 		static if (i < T.length) {
517 			static if (PRED!(T[i])) alias impl = TypeTuple!(i, impl!(i+1));
518 			else alias impl = impl!(i+1);
519 		} else alias impl = TypeTuple!();
520 	}
521 	alias indicesOf = impl!0;
522 }
523 private template FilterToType(alias PRED, T...) {
524 	import std.typetuple;
525 	template impl(size_t i) {
526 		static if (i < T.length) {
527 			static if (PRED!(T[i])) alias impl = TypeTuple!(typeof(T[i]), impl!(i+1));
528 			else alias impl = impl!(i+1);
529 		} else alias impl = TypeTuple!();
530 	}
531 	alias FilterToType = impl!0;
532 }
533 private template isRedisType(alias F) { enum isRedisType = is(typeof(&toRedis!(typeof(F)))); }
534 private template isNonRedisType(alias F) { enum isNonRedisType = !isRedisType!F; }
535 static assert(isRedisType!(int.init) && isRedisType!(string.init));
536 static assert(!isRedisType!((float[]).init));
537 
538 private template isRedisTypeAndNotID(alias F) { import std.algorithm; enum isRedisTypeAndNotID = !__traits(identifier, F).among("_id", "id") && isRedisType!F; }
539 private template isNonRedisTypeOrID(alias F) { enum isNonRedisTypeOrID = !isRedisTypeAndNotID!F; }
540 static assert(isRedisTypeAndNotID!(int.init) && isRedisTypeAndNotID!(string.init));
541 
542 private auto toTuple(size_t N, T)(T[N] values)
543 {
544 	import std.typecons;
545 	import std.typetuple;
546 	template impl(size_t i) {
547 		static if (i < N) alias impl = TypeTuple!(T, impl!(i+1));
548 		else alias impl = TypeTuple!();
549 	}
550 	Tuple!(impl!0) ret;
551 	foreach (i, T; impl!0) ret[i] = values[i];
552 	return ret;
553 }
554 
555 private template fieldNames(T)
556 {
557 	import std.typetuple;
558 	template impl(size_t i) {
559 		static if (i < T.tupleof.length)
560 			alias impl = TypeTuple!(__traits(identifier, T.tupleof[i]), impl!(i+1));
561 		else alias impl = TypeTuple!();
562 	}
563 	enum string[T.tupleof.length] fieldNames = [impl!0];
564 }
565 
566 unittest {
567 	static struct Test { int a; float b; void method() {} Test[] c; void opAssign(Test) {}; ~this() {} }
568 	static assert(fieldNames!Test[] == ["a", "b", "c"]);
569 }
570 
571 private template Replicate(T, size_t L)
572 {
573 	import std.typetuple;
574 	static if (L > 0) {
575 		alias Replicate = TypeTuple!(T, Replicate!(T, L-1));
576 	} else alias Replicate = TypeTuple!();
577 }