1 /** 2 Type safe implementations of common Redis storage idioms. 3 4 Note that the API is still subject to change! 5 6 Copyright: © 2014 Sönke Ludwig 7 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 8 Authors: Sönke Ludwig 9 */ 10 module vibe.db.redis.idioms; 11 12 import vibe.db.redis.redis; 13 import vibe.db.redis.types; 14 15 import core.time : msecs, seconds; 16 import std.typecons : Tuple; 17 18 19 /** 20 */ 21 struct RedisCollection(T /*: RedisValue*/, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 22 { 23 static assert(ID_LENGTH > 0, "IDs must have a length of at least one."); 24 static assert(!(OPTIONS & RedisCollectionOptions.supportIteration) || ID_LENGTH == 1, "ID generation currently not supported for ID lengths greater 2."); 25 26 alias IDS = Replicate!(long, ID_LENGTH); 27 static if (ID_LENGTH == 1) alias IDType = long; 28 else alias IDType = Tuple!IDS; 29 30 private { 31 RedisDatabase m_db; 32 string[ID_LENGTH] m_prefix; 33 string m_suffix; 34 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) { 35 @property string m_idCounter() const { return m_prefix[0] ~ "max"; } 36 @property string m_allSet() const { return m_prefix[0] ~ "all"; } 37 } 38 } 39 40 this(RedisDatabase db, Replicate!(string, ID_LENGTH) name, string suffix = null) 41 { 42 initialize(db, name, suffix); 43 } 44 45 void initialize(RedisDatabase db, Replicate!(string, ID_LENGTH) name, string suffix = null) 46 { 47 m_db = db; 48 foreach (i, N; name) { 49 if (i == 0) m_prefix[i] = name[i] ~ ":"; 50 else m_prefix[i] = ":" ~ name[i] ~ ":"; 51 } 52 if (suffix.length) m_suffix = ":" ~ suffix; 53 } 54 55 @property inout(RedisDatabase) database() inout { return m_db; } 56 57 T opIndex(IDS id) { return T(m_db, getKey(id)); } 58 59 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) { 60 /** Creates an ID without setting a corresponding value. 61 */ 62 IDType createID() 63 { 64 auto id = m_db.incr(m_idCounter); 65 static if (OPTIONS & RedisCollectionOptions.supportPaging) 66 m_db.zadd(m_allSet, id, id); 67 else m_db.sadd(m_allSet, id); 68 return id; 69 } 70 71 IDType add(U)(U args) 72 { 73 auto id = createID(); 74 this[id] = args; 75 return id; 76 } 77 78 bool isMember(long id) 79 { 80 static if (OPTIONS & RedisCollectionOptions.supportPaging) 81 return m_db.zscore(m_allSet, id).hasNext(); 82 else return m_db.sisMember(m_allSet, id); 83 } 84 85 static if (OPTIONS & RedisCollectionOptions.supportPaging) { 86 // TODO: add range queries 87 } 88 89 int opApply(int delegate(long id) del) 90 { 91 static if (OPTIONS & RedisCollectionOptions.supportPaging) { 92 foreach (id; m_db.zrange!long(m_allSet, 0, -1)) 93 if (auto ret = del(id)) 94 return ret; 95 } else { 96 foreach (id; m_db.smembers!long(m_allSet)) 97 if (auto ret = del(id)) 98 return ret; 99 } 100 return 0; 101 } 102 103 int opApply(int delegate(long id, T) del) 104 { 105 static if (OPTIONS & RedisCollectionOptions.supportPaging) { 106 foreach (id; m_db.zrange!long(m_allSet, 0, -1)) 107 if (auto ret = del(id, this[id])) 108 return ret; 109 } else { 110 foreach (id; m_db.smembers!long(m_allSet)) 111 if (auto ret = del(id, this[id])) 112 return ret; 113 } 114 return 0; 115 } 116 } 117 118 /** Removes an ID along with the corresponding value. 119 */ 120 void remove(IDS id) 121 { 122 this[id].remove(); 123 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) { 124 static if (OPTIONS & RedisCollectionOptions.supportPaging) 125 m_db.zrem(m_allSet, id); 126 else 127 m_db.srem(m_allSet, id); 128 } 129 } 130 131 132 private string getKey(IDS ids) 133 { 134 import std.conv; 135 static if (ID_LENGTH == 1) { 136 return m_prefix[0] ~ ids.to!string ~ m_suffix; 137 } else { 138 string ret; 139 foreach (i, id; ids) ret ~= m_prefix[i] ~ id.to!string; 140 return ret ~ m_suffix; 141 } 142 } 143 } 144 145 enum RedisCollectionOptions { 146 none = 0, // Plain collection without iteration/paging support 147 supportIteration = 1<<0, // Store IDs in a set to be able to iterate and check for existence 148 supportPaging = 1<<1, // Store IDs in a sorted set, to support range based queries 149 defaults = supportIteration 150 } 151 152 153 /** Models a set of numbered hashes. 154 155 This structure is roughly equivalent to a $(D string[string][long]) and is 156 commonly used to store collections of objects, such as all users of a 157 service. For a strongly typed variant of this class, see 158 $(D RedisObjectCollection). 159 160 See_also: $(D RedisObjectCollection) 161 */ 162 template RedisHashCollection(RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 163 { 164 alias RedisHashCollection = RedisCollection!(RedisHash, OPTIONS, ID_LENGTH); 165 } 166 167 168 /** Models a strongly typed set of numbered hashes. 169 170 This structure is roughly equivalent of a $(D T[long]). 171 172 See_also: $(D RedisHashCollection) 173 */ 174 template RedisObjectCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 175 { 176 alias RedisObjectCollection = RedisCollection!(RedisObject!T, OPTIONS, ID_LENGTH); 177 } 178 179 /// 180 unittest { 181 struct User { 182 string name; 183 string email; 184 int age; 185 string password; 186 } 187 188 void test() 189 { 190 auto db = connectRedis("127.0.0.1").getDatabase(0); 191 db.deleteAll(); 192 193 auto users = RedisObjectCollection!User(db, "users"); 194 assert(users.add(User("Tom", "tom@example.com", 42, "secret")) == 0); 195 assert(users.add(User("Peter", "peter@example.com", 42, "secret")) == 1); 196 197 auto peter = users[1]; 198 assert(peter.name == "Peter"); 199 } 200 } 201 202 203 /** Models a single strongly typed object. 204 205 This structure is rougly equivalent to a value of type $(D T). The 206 underlying data is represented as a Redis hash. This means that only 207 primitive fields are supported for $(D T). 208 */ 209 struct RedisObject(T) { 210 private { 211 RedisHash!string m_hash; 212 } 213 214 this(RedisDatabase db, string key) 215 { 216 m_hash = RedisHash!string(db, key); 217 } 218 219 this(RedisHash!string hash) 220 { 221 m_hash = hash; 222 } 223 224 @property T get() 225 { 226 T ret; 227 auto repl = m_hash.database.hmget(m_hash.key, keys); 228 foreach (i, F; typeof(ret.tupleof)) { 229 assert(!repl.empty); 230 __traits(getMember, ret, keys[i]) = repl.front.fromRedis!F; 231 repl.popFront(); 232 } 233 assert(repl.empty); 234 return ret; 235 } 236 237 @property bool exists() { return m_hash.value.exists(); } 238 239 alias get this; 240 241 void remove() { m_hash.remove(); } 242 243 void opAssign(T val) 244 { 245 m_hash.database.hmset(m_hash.key, toTuple(toKeysAndValues(val)).expand); 246 } 247 248 mixin(fields()); 249 250 static private string fields() 251 { 252 string ret; 253 foreach (name; keys) { 254 ret ~= "@property auto "~name~"() { return RedisObjectField!(typeof(T."~name~"))(m_hash, \""~name~"\"); }\n"; 255 ret ~= "@property void "~name~"(typeof(T."~name~") val) { this."~name~".opAssign(val); }\n"; 256 } 257 return ret; 258 } 259 260 /*@property auto opDispatch(string name)() //if (is(typeof(getMember, T.init, name))) 261 { 262 return RedisObjectField!(typeof(__traits(getMember, T.init, name)))(m_hash, name); 263 }*/ 264 265 private static string[T.tupleof.length*2] toKeysAndValues(T val) 266 { 267 string[T.tupleof.length*2] ret; 268 enum keys = fieldNames!T; 269 foreach (i, m; val.tupleof) { 270 ret[i*2+0] = keys[i]; 271 ret[i*2+1] = m.toRedis(); 272 } 273 return ret; 274 } 275 276 private enum keys = fieldNames!T; 277 } 278 279 struct RedisObjectField(T) { 280 private { 281 RedisHash!string m_hash; 282 string m_field; 283 } 284 285 this(RedisHash!string hash, string field) 286 { 287 m_hash = hash; 288 m_field = field; 289 } 290 291 @property T get() { return m_hash.database.hget!string(m_hash.key, m_field).fromRedis!T; } 292 293 alias get this; 294 295 void opAssign(T val) { m_hash.database.hset(m_hash.key, m_field, val.toRedis); } 296 297 void opUnary(string op)() if(op == "++") { m_hash.database.hincr(m_hash.key, m_field, 1); } 298 void opUnary(string op)() if(op == "--") { m_hash.database.hincr(m_hash.key, m_field, -1); } 299 300 void opOpAssign(string op)(long val) if (op == "+") { m_hash.database.hincr(m_hash.key, m_field, val); } 301 void opOpAssign(string op)(long val) if (op == "-") { m_hash.database.hincr(m_hash.key, m_field, -val); } 302 void opOpAssign(string op)(double val) if (op == "+") { m_hash.database.hincr(m_hash.key, m_field, val); } 303 void opOpAssign(string op)(double val) if (op == "-") { m_hash.database.hincr(m_hash.key, m_field, -val); } 304 } 305 306 307 /** Models a strongly typed numbered set of values. 308 309 310 */ 311 template RedisSetCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 312 { 313 alias RedisSetCollection = RedisCollection!(RedisSet!T, OPTIONS, ID_LENGTH); 314 } 315 316 /// 317 unittest { 318 void test() 319 { 320 auto db = connectRedis("127.0.0.1").getDatabase(0); 321 auto user_groups = RedisSetCollection!(string, RedisCollectionOptions.none)(db, "user_groups"); 322 323 // add some groups for user with ID 0 324 user_groups[0].insert("cooking"); 325 user_groups[0].insert("hiking"); 326 // add some groups for user with ID 1 327 user_groups[1].insert("coding"); 328 329 assert(user_groups[0].contains("hiking")); 330 assert(!user_groups[0].contains("coding")); 331 assert(user_groups[1].contains("coding")); 332 333 user_groups[0].remove("hiking"); 334 assert(!user_groups[0].contains("hiking")); 335 } 336 } 337 338 339 /** Models a strongly typed numbered set of values. 340 341 342 */ 343 template RedisListCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 344 { 345 alias RedisListCollection = RedisCollection!(RedisList!T, OPTIONS, ID_LENGTH); 346 } 347 348 349 /** Models a strongly typed numbered set of values. 350 351 352 */ 353 template RedisStringCollection(T = string, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 354 { 355 alias RedisStringCollection = RedisCollection!(RedisString!T, OPTIONS, ID_LENGTH); 356 } 357 358 359 // TODO: support distributed locking 360 struct RedisLock { 361 private { 362 RedisDatabase m_db; 363 string m_key; 364 string m_scriptSHA; 365 } 366 367 this(RedisDatabase db, string lock_key) 368 { 369 m_db = db; 370 m_key = lock_key; 371 m_scriptSHA = m_db.scriptLoad( 372 `if redis.call("get",KEYS[1]) == ARGV[1] then 373 return redis.call("del",KEYS[1]) 374 else 375 return 0 376 end`); 377 } 378 379 void performLocked(scope void delegate() del) 380 { 381 import std.random; 382 import vibe.core.core; 383 import vibe.data.bson; 384 385 auto lockval = BsonObjectID.generate(); 386 while (!m_db.setNX(m_key, cast(ubyte[])lockval, 30.seconds)) 387 sleep(uniform(1, 50).msecs); 388 389 scope (exit) m_db.evalSHA!(string, ubyte[])(m_scriptSHA, [m_key], cast(ubyte[])lockval); 390 391 del(); 392 } 393 } 394 395 396 // utility structure, temporarily placed here 397 struct JsonEncoded(T) { 398 import vibe.data.json; 399 T value; 400 401 alias value this; 402 403 static JsonEncoded fromString(string str) { return JsonEncoded(deserializeJson!T(str)); } 404 string toString() { return serializeToJsonString(value); } 405 406 static assert(isStringSerializable!JsonEncoded); 407 } 408 JsonEncoded!T jsonEncoded(T)(T value) { return JsonEncoded!T(value); } 409 410 411 // utility structure, temporarily placed here 412 struct LazyString(T...) { 413 private { 414 T m_values; 415 } 416 417 this(T values) { m_values = values; } 418 419 void toString(void delegate(string) sink) 420 { 421 foreach (v; m_values) 422 dst.formattedWrite("%s", v); 423 } 424 } 425 426 427 /** 428 Strips all non-Redis fields from a struct. 429 430 The returned struct will contain only fiels that can be converted using 431 $(D toRedis) and that have names different than "id" or "_id". 432 433 To reconstruct the full struct type, use the $(D RedisStripped.unstrip) 434 method. 435 */ 436 RedisStripped!(T, strip_id) redisStrip(bool strip_id = true, T)(in T val) { return RedisStripped!(T, strip_id)(val); } 437 438 /** 439 Represents the stripped type of a struct. 440 441 Strips all fields that cannot be directly stored as values in the Redis 442 database. By default, any field named `id` or `_id` is also stripped. Set 443 the `strip_id` parameter to `false` to keep those fields. 444 445 See_also: $(D redisStrip) 446 */ 447 struct RedisStripped(T, bool strip_id = true) { 448 import std.traits : Select, select; 449 import std.typetuple; 450 451 //pragma(msg, membersString!()); 452 mixin(membersString()); 453 454 alias StrippedMembers = FilterToType!(Select!(strip_id, isNonRedisTypeOrID, isNonRedisType), T.tupleof); 455 alias UnstrippedMembers = FilterToType!(Select!(strip_id, isRedisTypeAndNotID, isRedisType), T.tupleof); 456 alias strippedMemberIndices = indicesOf!(Select!(strip_id, isNonRedisTypeOrID, isNonRedisType), T.tupleof); 457 alias unstrippedMemberIndices = indicesOf!(Select!(strip_id, isRedisTypeAndNotID, isRedisType), T.tupleof); 458 459 this(in T src) { foreach (i, idx; unstrippedMemberIndices) this.tupleof[i] = src.tupleof[idx]; } 460 461 /** Reconstructs the full (unstripped) struct value. 462 463 The parameters for this method are all stripped fields in the order in 464 which they appear in the original struct definition. 465 */ 466 T unstrip(StrippedMembers stripped_members) { 467 T ret; 468 populateRedisFields(ret, this.tupleof); 469 populateNonRedisFields(ret, stripped_members); 470 return ret; 471 } 472 473 private void populateRedisFields(ref T dst, UnstrippedMembers values) 474 { 475 foreach (i, v; values) 476 dst.tupleof[unstrippedMemberIndices[i]] = v; 477 } 478 479 private void populateNonRedisFields(ref T dst, StrippedMembers values) 480 { 481 foreach (i, v; values) 482 dst.tupleof[strippedMemberIndices[i]] = v; 483 } 484 485 486 /*pragma(msg, T); 487 pragma(msg, "stripped: "~StrippedMembers.stringof~" - "~strippedMemberIndices.stringof); 488 pragma(msg, "unstripped: "~UnstrippedMembers.stringof~" - "~unstrippedMemberIndices.stringof);*/ 489 490 private static string membersString() 491 { 492 string ret; 493 foreach (idx; unstrippedMemberIndices) { 494 enum name = __traits(identifier, T.tupleof[idx]); 495 ret ~= "typeof(T."~name~") "~name~";\n"; 496 } 497 return ret; 498 } 499 } 500 501 unittest { 502 static struct S1 { int id; string field; string[] array; } 503 auto s1 = S1(42, "hello", ["world"]); 504 auto s1s = redisStrip(s1); 505 static assert(!is(typeof(s1s.id))); 506 static assert(is(typeof(s1s.field))); 507 static assert(!is(typeof(s1s.array))); 508 assert(s1s.field == "hello"); 509 auto s1u = s1s.unstrip(42, ["world"]); 510 assert(s1u == s1); 511 } 512 513 private template indicesOf(alias PRED, T...) 514 { 515 import std.typetuple; 516 template impl(size_t i) { 517 static if (i < T.length) { 518 static if (PRED!(T[i])) alias impl = TypeTuple!(i, impl!(i+1)); 519 else alias impl = impl!(i+1); 520 } else alias impl = TypeTuple!(); 521 } 522 alias indicesOf = impl!0; 523 } 524 private template FilterToType(alias PRED, T...) { 525 import std.typetuple; 526 template impl(size_t i) { 527 static if (i < T.length) { 528 static if (PRED!(T[i])) alias impl = TypeTuple!(typeof(T[i]), impl!(i+1)); 529 else alias impl = impl!(i+1); 530 } else alias impl = TypeTuple!(); 531 } 532 alias FilterToType = impl!0; 533 } 534 private template isRedisType(alias F) { enum isRedisType = is(typeof(&toRedis!(typeof(F)))); } 535 private template isNonRedisType(alias F) { enum isNonRedisType = !isRedisType!F; } 536 static assert(isRedisType!(int.init) && isRedisType!(string.init)); 537 static assert(!isRedisType!((float[]).init)); 538 539 private template isRedisTypeAndNotID(alias F) { import std.algorithm; enum isRedisTypeAndNotID = !__traits(identifier, F).among("_id", "id") && isRedisType!F; } 540 private template isNonRedisTypeOrID(alias F) { enum isNonRedisTypeOrID = !isRedisTypeAndNotID!F; } 541 static assert(isRedisTypeAndNotID!(int.init) && isRedisTypeAndNotID!(string.init)); 542 543 private auto toTuple(size_t N, T)(T[N] values) 544 { 545 import std.typecons; 546 import std.typetuple; 547 template impl(size_t i) { 548 static if (i < N) alias impl = TypeTuple!(T, impl!(i+1)); 549 else alias impl = TypeTuple!(); 550 } 551 Tuple!(impl!0) ret; 552 foreach (i, T; impl!0) ret[i] = values[i]; 553 return ret; 554 } 555 556 private template fieldNames(T) 557 { 558 import std.typetuple; 559 template impl(size_t i) { 560 static if (i < T.tupleof.length) 561 alias impl = TypeTuple!(__traits(identifier, T.tupleof[i]), impl!(i+1)); 562 else alias impl = TypeTuple!(); 563 } 564 enum string[T.tupleof.length] fieldNames = [impl!0]; 565 } 566 567 unittest { 568 static struct Test { int a; float b; void method() {} Test[] c; void opAssign(Test) {}; ~this() {} } 569 static assert(fieldNames!Test[] == ["a", "b", "c"]); 570 } 571 572 private template Replicate(T, size_t L) 573 { 574 import std.typetuple; 575 static if (L > 0) { 576 alias Replicate = TypeTuple!(T, Replicate!(T, L-1)); 577 } else alias Replicate = TypeTuple!(); 578 }