1 /**
2 Type safe implementations of common Redis storage idioms.
3
4 Note that the API is still subject to change!
5
6 Copyright: © 2014 Sönke Ludwig
7 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
8 Authors: Sönke Ludwig
9 */
10 module vibe.db.redis.idioms;
11
12 import vibe.db.redis.redis;
13 import vibe.db.redis.types;
14
15 import core.time : msecs, seconds;
16 import std.typecons : Tuple;
17
18
19 /**
20 */
21 struct RedisCollection(T /*: RedisValue*/, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
22 {
23 static assert(ID_LENGTH > 0, "IDs must have a length of at least one.");
24 static assert(!(OPTIONS & RedisCollectionOptions.supportIteration) || ID_LENGTH == 1, "ID generation currently not supported for ID lengths greater 2.");
25
26 alias IDS = Replicate!(long, ID_LENGTH);
27 static if (ID_LENGTH == 1) alias IDType = long;
28 else alias IDType = Tuple!IDS;
29
30 private {
31 RedisDatabase m_db;
32 string[ID_LENGTH] m_prefix;
33 string m_suffix;
34 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) {
35 @property string m_idCounter() const { return m_prefix[0] ~ "max"; }
36 @property string m_allSet() const { return m_prefix[0] ~ "all"; }
37 }
38 }
39
40 this(RedisDatabase db, Replicate!(string, ID_LENGTH) name, string suffix = null)
41 {
42 initialize(db, name, suffix);
43 }
44
45 void initialize(RedisDatabase db, Replicate!(string, ID_LENGTH) name, string suffix = null)
46 {
47 m_db = db;
48 foreach (i, N; name) {
49 if (i == 0) m_prefix[i] = name[i] ~ ":";
50 else m_prefix[i] = ":" ~ name[i] ~ ":";
51 }
52 if (suffix.length) m_suffix = ":" ~ suffix;
53 }
54
55 @property inout(RedisDatabase) database() inout { return m_db; }
56
57 T opIndex(IDS id) { return T(m_db, getKey(id)); }
58
59 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) {
60 /** Creates an ID without setting a corresponding value.
61 */
62 IDType createID()
63 {
64 auto id = m_db.incr(m_idCounter);
65 static if (OPTIONS & RedisCollectionOptions.supportPaging)
66 m_db.zadd(m_allSet, id, id);
67 else m_db.sadd(m_allSet, id);
68 return id;
69 }
70
71 IDType add(U)(U args)
72 {
73 auto id = createID();
74 this[id] = args;
75 return id;
76 }
77
78 bool isMember(long id)
79 {
80 static if (OPTIONS & RedisCollectionOptions.supportPaging)
81 return m_db.zscore(m_allSet, id).hasNext();
82 else return m_db.sisMember(m_allSet, id);
83 }
84
85 static if (OPTIONS & RedisCollectionOptions.supportPaging) {
86 // TODO: add range queries
87 }
88
89 int opApply(int delegate(long id) del)
90 {
91 static if (OPTIONS & RedisCollectionOptions.supportPaging) {
92 foreach (id; m_db.zrange!long(m_allSet, 0, -1))
93 if (auto ret = del(id))
94 return ret;
95 } else {
96 foreach (id; m_db.smembers!long(m_allSet))
97 if (auto ret = del(id))
98 return ret;
99 }
100 return 0;
101 }
102
103 int opApply(int delegate(long id, T) del)
104 {
105 static if (OPTIONS & RedisCollectionOptions.supportPaging) {
106 foreach (id; m_db.zrange!long(m_allSet, 0, -1))
107 if (auto ret = del(id, this[id]))
108 return ret;
109 } else {
110 foreach (id; m_db.smembers!long(m_allSet))
111 if (auto ret = del(id, this[id]))
112 return ret;
113 }
114 return 0;
115 }
116 }
117
118 /** Removes an ID along with the corresponding value.
119 */
120 void remove(IDS id)
121 {
122 this[id].remove();
123 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) {
124 static if (OPTIONS & RedisCollectionOptions.supportPaging)
125 m_db.zrem(m_allSet, id);
126 else
127 m_db.srem(m_allSet, id);
128 }
129 }
130
131
132 private string getKey(IDS ids)
133 {
134 import std.conv;
135 static if (ID_LENGTH == 1) {
136 return m_prefix[0] ~ ids.to!string ~ m_suffix;
137 } else {
138 string ret;
139 foreach (i, id; ids) ret ~= m_prefix[i] ~ id.to!string;
140 return ret ~ m_suffix;
141 }
142 }
143 }
144
145 enum RedisCollectionOptions {
146 none = 0, // Plain collection without iteration/paging support
147 supportIteration = 1<<0, // Store IDs in a set to be able to iterate and check for existence
148 supportPaging = 1<<1, // Store IDs in a sorted set, to support range based queries
149 defaults = supportIteration
150 }
151
152
153 /** Models a set of numbered hashes.
154
155 This structure is roughly equivalent to a $(D string[string][long]) and is
156 commonly used to store collections of objects, such as all users of a
157 service. For a strongly typed variant of this class, see
158 $(D RedisObjectCollection).
159
160 See_also: $(D RedisObjectCollection)
161 */
162 template RedisHashCollection(RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
163 {
164 alias RedisHashCollection = RedisCollection!(RedisHash, OPTIONS, ID_LENGTH);
165 }
166
167
168 /** Models a strongly typed set of numbered hashes.
169
170 This structure is roughly equivalent of a $(D T[long]).
171
172 See_also: $(D RedisHashCollection)
173 */
174 template RedisObjectCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
175 {
176 alias RedisObjectCollection = RedisCollection!(RedisObject!T, OPTIONS, ID_LENGTH);
177 }
178
179 ///
180 unittest {
181 struct User {
182 string name;
183 string email;
184 int age;
185 string password;
186 }
187
188 void test()
189 {
190 auto db = connectRedis("127.0.0.1").getDatabase(0);
191 db.deleteAll();
192
193 auto users = RedisObjectCollection!User(db, "users");
194 assert(users.add(User("Tom", "tom@example.com", 42, "secret")) == 0);
195 assert(users.add(User("Peter", "peter@example.com", 42, "secret")) == 1);
196
197 auto peter = users[1];
198 assert(peter.name == "Peter");
199 }
200 }
201
202
203 /** Models a single strongly typed object.
204
205 This structure is rougly equivalent to a value of type $(D T). The
206 underlying data is represented as a Redis hash. This means that only
207 primitive fields are supported for $(D T).
208 */
209 struct RedisObject(T) {
210 private {
211 RedisHash!string m_hash;
212 }
213
214 this(RedisDatabase db, string key)
215 {
216 m_hash = RedisHash!string(db, key);
217 }
218
219 this(RedisHash!string hash)
220 {
221 m_hash = hash;
222 }
223
224 @property T get()
225 {
226 T ret;
227 auto repl = m_hash.database.hmget(m_hash.key, keys);
228 foreach (i, F; typeof(ret.tupleof)) {
229 assert(!repl.empty);
230 __traits(getMember, ret, keys[i]) = repl.front.fromRedis!F;
231 repl.popFront();
232 }
233 assert(repl.empty);
234 return ret;
235 }
236
237 @property bool exists() { return m_hash.value.exists(); }
238
239 alias get this;
240
241 void remove() { m_hash.remove(); }
242
243 void opAssign(T val)
244 {
245 m_hash.database.hmset(m_hash.key, toTuple(toKeysAndValues(val)).expand);
246 }
247
248 mixin(fields());
249
250 static private string fields()
251 {
252 string ret;
253 foreach (name; keys) {
254 ret ~= "@property auto "~name~"() { return RedisObjectField!(typeof(T."~name~"))(m_hash, \""~name~"\"); }\n";
255 ret ~= "@property void "~name~"(typeof(T."~name~") val) { this."~name~".opAssign(val); }\n";
256 }
257 return ret;
258 }
259
260 /*@property auto opDispatch(string name)() //if (is(typeof(getMember, T.init, name)))
261 {
262 return RedisObjectField!(typeof(__traits(getMember, T.init, name)))(m_hash, name);
263 }*/
264
265 private static string[T.tupleof.length*2] toKeysAndValues(T val)
266 {
267 string[T.tupleof.length*2] ret;
268 enum keys = fieldNames!T;
269 foreach (i, m; val.tupleof) {
270 ret[i*2+0] = keys[i];
271 ret[i*2+1] = m.toRedis();
272 }
273 return ret;
274 }
275
276 private enum keys = fieldNames!T;
277 }
278
279 struct RedisObjectField(T) {
280 private {
281 RedisHash!string m_hash;
282 string m_field;
283 }
284
285 this(RedisHash!string hash, string field)
286 {
287 m_hash = hash;
288 m_field = field;
289 }
290
291 @property T get() { return m_hash.database.hget!string(m_hash.key, m_field).fromRedis!T; }
292
293 alias get this;
294
295 void opAssign(T val) { m_hash.database.hset(m_hash.key, m_field, val.toRedis); }
296
297 void opUnary(string op)() if(op == "++") { m_hash.database.hincr(m_hash.key, m_field, 1); }
298 void opUnary(string op)() if(op == "--") { m_hash.database.hincr(m_hash.key, m_field, -1); }
299
300 void opOpAssign(string op)(long val) if (op == "+") { m_hash.database.hincr(m_hash.key, m_field, val); }
301 void opOpAssign(string op)(long val) if (op == "-") { m_hash.database.hincr(m_hash.key, m_field, -val); }
302 void opOpAssign(string op)(double val) if (op == "+") { m_hash.database.hincr(m_hash.key, m_field, val); }
303 void opOpAssign(string op)(double val) if (op == "-") { m_hash.database.hincr(m_hash.key, m_field, -val); }
304 }
305
306
307 /** Models a strongly typed numbered set of values.
308
309
310 */
311 template RedisSetCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
312 {
313 alias RedisSetCollection = RedisCollection!(RedisSet!T, OPTIONS, ID_LENGTH);
314 }
315
316 ///
317 unittest {
318 void test()
319 {
320 auto db = connectRedis("127.0.0.1").getDatabase(0);
321 auto user_groups = RedisSetCollection!(string, RedisCollectionOptions.none)(db, "user_groups");
322
323 // add some groups for user with ID 0
324 user_groups[0].insert("cooking");
325 user_groups[0].insert("hiking");
326 // add some groups for user with ID 1
327 user_groups[1].insert("coding");
328
329 assert(user_groups[0].contains("hiking"));
330 assert(!user_groups[0].contains("coding"));
331 assert(user_groups[1].contains("coding"));
332
333 user_groups[0].remove("hiking");
334 assert(!user_groups[0].contains("hiking"));
335 }
336 }
337
338
339 /** Models a strongly typed numbered set of values.
340
341
342 */
343 template RedisListCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
344 {
345 alias RedisListCollection = RedisCollection!(RedisList!T, OPTIONS, ID_LENGTH);
346 }
347
348
349 /** Models a strongly typed numbered set of values.
350
351
352 */
353 template RedisStringCollection(T = string, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1)
354 {
355 alias RedisStringCollection = RedisCollection!(RedisString!T, OPTIONS, ID_LENGTH);
356 }
357
358
359 // TODO: support distributed locking
360 struct RedisLock {
361 private {
362 RedisDatabase m_db;
363 string m_key;
364 string m_scriptSHA;
365 }
366
367 this(RedisDatabase db, string lock_key)
368 {
369 m_db = db;
370 m_key = lock_key;
371 m_scriptSHA = m_db.scriptLoad(
372 `if redis.call("get",KEYS[1]) == ARGV[1] then
373 return redis.call("del",KEYS[1])
374 else
375 return 0
376 end`);
377 }
378
379 void performLocked(scope void delegate() del)
380 {
381 import std.random;
382 import vibe.core.core;
383 import vibe.data.bson;
384
385 auto lockval = BsonObjectID.generate();
386 while (!m_db.setNX(m_key, cast(ubyte[])lockval, 30.seconds))
387 sleep(uniform(1, 50).msecs);
388
389 scope (exit) m_db.evalSHA!(string, ubyte[])(m_scriptSHA, [m_key], cast(ubyte[])lockval);
390
391 del();
392 }
393 }
394
395
396 // utility structure, temporarily placed here
397 struct JsonEncoded(T) {
398 import vibe.data.json;
399 T value;
400
401 alias value this;
402
403 static JsonEncoded fromString(string str) { return JsonEncoded(deserializeJson!T(str)); }
404 string toString() { return serializeToJsonString(value); }
405
406 static assert(isStringSerializable!JsonEncoded);
407 }
408 JsonEncoded!T jsonEncoded(T)(T value) { return JsonEncoded!T(value); }
409
410
411 // utility structure, temporarily placed here
412 struct LazyString(T...) {
413 private {
414 T m_values;
415 }
416
417 this(T values) { m_values = values; }
418
419 void toString(void delegate(string) sink)
420 {
421 foreach (v; m_values)
422 dst.formattedWrite("%s", v);
423 }
424 }
425
426
427 /**
428 Strips all non-Redis fields from a struct.
429
430 The returned struct will contain only fiels that can be converted using
431 $(D toRedis) and that have names different than "id" or "_id".
432
433 To reconstruct the full struct type, use the $(D RedisStripped.unstrip)
434 method.
435 */
436 RedisStripped!(T, strip_id) redisStrip(bool strip_id = true, T)(in T val) { return RedisStripped!(T, strip_id)(val); }
437
438 /**
439 Represents the stripped type of a struct.
440
441 Strips all fields that cannot be directly stored as values in the Redis
442 database. By default, any field named `id` or `_id` is also stripped. Set
443 the `strip_id` parameter to `false` to keep those fields.
444
445 See_also: $(D redisStrip)
446 */
447 struct RedisStripped(T, bool strip_id = true) {
448 import std.traits : Select, select;
449 import std.typetuple;
450
451 //pragma(msg, membersString!());
452 mixin(membersString());
453
454 alias StrippedMembers = FilterToType!(Select!(strip_id, isNonRedisTypeOrID, isNonRedisType), T.tupleof);
455 alias UnstrippedMembers = FilterToType!(Select!(strip_id, isRedisTypeAndNotID, isRedisType), T.tupleof);
456 alias strippedMemberIndices = indicesOf!(Select!(strip_id, isNonRedisTypeOrID, isNonRedisType), T.tupleof);
457 alias unstrippedMemberIndices = indicesOf!(Select!(strip_id, isRedisTypeAndNotID, isRedisType), T.tupleof);
458
459 this(in T src) { foreach (i, idx; unstrippedMemberIndices) this.tupleof[i] = src.tupleof[idx]; }
460
461 /** Reconstructs the full (unstripped) struct value.
462
463 The parameters for this method are all stripped fields in the order in
464 which they appear in the original struct definition.
465 */
466 T unstrip(StrippedMembers stripped_members) {
467 T ret;
468 populateRedisFields(ret, this.tupleof);
469 populateNonRedisFields(ret, stripped_members);
470 return ret;
471 }
472
473 private void populateRedisFields(ref T dst, UnstrippedMembers values)
474 {
475 foreach (i, v; values)
476 dst.tupleof[unstrippedMemberIndices[i]] = v;
477 }
478
479 private void populateNonRedisFields(ref T dst, StrippedMembers values)
480 {
481 foreach (i, v; values)
482 dst.tupleof[strippedMemberIndices[i]] = v;
483 }
484
485
486 /*pragma(msg, T);
487 pragma(msg, "stripped: "~StrippedMembers.stringof~" - "~strippedMemberIndices.stringof);
488 pragma(msg, "unstripped: "~UnstrippedMembers.stringof~" - "~unstrippedMemberIndices.stringof);*/
489
490 private static string membersString()
491 {
492 string ret;
493 foreach (idx; unstrippedMemberIndices) {
494 enum name = __traits(identifier, T.tupleof[idx]);
495 ret ~= "typeof(T."~name~") "~name~";\n";
496 }
497 return ret;
498 }
499 }
500
501 unittest {
502 static struct S1 { int id; string field; string[] array; }
503 auto s1 = S1(42, "hello", ["world"]);
504 auto s1s = redisStrip(s1);
505 static assert(!is(typeof(s1s.id)));
506 static assert(is(typeof(s1s.field)));
507 static assert(!is(typeof(s1s.array)));
508 assert(s1s.field == "hello");
509 auto s1u = s1s.unstrip(42, ["world"]);
510 assert(s1u == s1);
511 }
512
513 private template indicesOf(alias PRED, T...)
514 {
515 import std.typetuple;
516 template impl(size_t i) {
517 static if (i < T.length) {
518 static if (PRED!(T[i])) alias impl = TypeTuple!(i, impl!(i+1));
519 else alias impl = impl!(i+1);
520 } else alias impl = TypeTuple!();
521 }
522 alias indicesOf = impl!0;
523 }
524 private template FilterToType(alias PRED, T...) {
525 import std.typetuple;
526 template impl(size_t i) {
527 static if (i < T.length) {
528 static if (PRED!(T[i])) alias impl = TypeTuple!(typeof(T[i]), impl!(i+1));
529 else alias impl = impl!(i+1);
530 } else alias impl = TypeTuple!();
531 }
532 alias FilterToType = impl!0;
533 }
534 private template isRedisType(alias F) { enum isRedisType = is(typeof(&toRedis!(typeof(F)))); }
535 private template isNonRedisType(alias F) { enum isNonRedisType = !isRedisType!F; }
536 static assert(isRedisType!(int.init) && isRedisType!(string.init));
537 static assert(!isRedisType!((float[]).init));
538
539 private template isRedisTypeAndNotID(alias F) { import std.algorithm; enum isRedisTypeAndNotID = !__traits(identifier, F).among("_id", "id") && isRedisType!F; }
540 private template isNonRedisTypeOrID(alias F) { enum isNonRedisTypeOrID = !isRedisTypeAndNotID!F; }
541 static assert(isRedisTypeAndNotID!(int.init) && isRedisTypeAndNotID!(string.init));
542
543 private auto toTuple(size_t N, T)(T[N] values)
544 {
545 import std.typecons;
546 import std.typetuple;
547 template impl(size_t i) {
548 static if (i < N) alias impl = TypeTuple!(T, impl!(i+1));
549 else alias impl = TypeTuple!();
550 }
551 Tuple!(impl!0) ret;
552 foreach (i, T; impl!0) ret[i] = values[i];
553 return ret;
554 }
555
556 private template fieldNames(T)
557 {
558 import std.typetuple;
559 template impl(size_t i) {
560 static if (i < T.tupleof.length)
561 alias impl = TypeTuple!(__traits(identifier, T.tupleof[i]), impl!(i+1));
562 else alias impl = TypeTuple!();
563 }
564 enum string[T.tupleof.length] fieldNames = [impl!0];
565 }
566
567 unittest {
568 static struct Test { int a; float b; void method() {} Test[] c; void opAssign(Test) {}; ~this() {} }
569 static assert(fieldNames!Test[] == ["a", "b", "c"]);
570 }
571
572 private template Replicate(T, size_t L)
573 {
574 import std.typetuple;
575 static if (L > 0) {
576 alias Replicate = TypeTuple!(T, Replicate!(T, L-1));
577 } else alias Replicate = TypeTuple!();
578 }