1 /** 2 Type safe implementations of common Redis storage idioms. 3 4 Note that the API is still subject to change! 5 6 Copyright: © 2014 RejectedSoftware e.K. 7 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 8 Authors: Sönke Ludwig 9 */ 10 module vibe.db.redis.idioms; 11 12 import vibe.db.redis.redis; 13 import vibe.db.redis.types; 14 15 import core.time : msecs, seconds; 16 17 18 /** 19 */ 20 struct RedisCollection(T /*: RedisValue*/, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 21 { 22 static assert(ID_LENGTH > 0, "IDs must have a length of at least one."); 23 static assert(!(OPTIONS & RedisCollectionOptions.supportIteration) || ID_LENGTH == 1, "ID generation currently not supported for ID lengths greater 2."); 24 25 alias IDS = Replicate!(long, ID_LENGTH); 26 static if (ID_LENGTH == 1) alias IDType = long; 27 else alias IDType = Tuple!IDS; 28 29 private { 30 RedisDatabase m_db; 31 string[ID_LENGTH] m_prefix; 32 string m_suffix; 33 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) { 34 @property string m_idCounter() const { return m_prefix[0] ~ "max"; } 35 @property string m_allSet() const { return m_prefix[0] ~ "all"; } 36 } 37 } 38 39 this(RedisDatabase db, Replicate!(string, ID_LENGTH) name, string suffix = null) 40 { 41 initialize(db, name, suffix); 42 } 43 44 void initialize(RedisDatabase db, Replicate!(string, ID_LENGTH) name, string suffix = null) 45 { 46 m_db = db; 47 foreach (i, N; name) { 48 if (i == 0) m_prefix[i] = name[i] ~ ":"; 49 else m_prefix[i] = ":" ~ name[i] ~ ":"; 50 } 51 if (suffix.length) m_suffix = ":" ~ suffix; 52 } 53 54 @property inout(RedisDatabase) database() inout { return m_db; } 55 56 T opIndex(IDS id) { return T(m_db, getKey(id)); } 57 58 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) { 59 /** Creates an ID without setting a corresponding value. 60 */ 61 IDType createID() 62 { 63 auto id = m_db.incr(m_idCounter); 64 static if (OPTIONS & RedisCollectionOptions.supportPaging) 65 m_db.zadd(m_allSet, id, id); 66 else m_db.sadd(m_allSet, id); 67 return id; 68 } 69 70 IDType add(U)(U args) 71 { 72 auto id = createID(); 73 this[id] = args; 74 return id; 75 } 76 77 bool isMember(long id) 78 { 79 static if (OPTIONS & RedisCollectionOptions.supportPaging) 80 return m_db.zscore(m_allSet, id).hasNext(); 81 else return m_db.sisMember(m_allSet, id); 82 } 83 84 static if (OPTIONS & RedisCollectionOptions.supportPaging) { 85 // TODO: add range queries 86 } 87 88 int opApply(int delegate(long id) del) 89 { 90 static if (OPTIONS & RedisCollectionOptions.supportPaging) { 91 foreach (id; m_db.zrange!long(m_allSet, 0, -1)) 92 if (auto ret = del(id)) 93 return ret; 94 } else { 95 foreach (id; m_db.smembers!long(m_allSet)) 96 if (auto ret = del(id)) 97 return ret; 98 } 99 return 0; 100 } 101 102 int opApply(int delegate(long id, T) del) 103 { 104 static if (OPTIONS & RedisCollectionOptions.supportPaging) { 105 foreach (id; m_db.zrange!long(m_allSet, 0, -1)) 106 if (auto ret = del(id, this[id])) 107 return ret; 108 } else { 109 foreach (id; m_db.smembers!long(m_allSet)) 110 if (auto ret = del(id, this[id])) 111 return ret; 112 } 113 return 0; 114 } 115 } 116 117 /** Removes an ID along with the corresponding value. 118 */ 119 void remove(IDS id) 120 { 121 this[id].remove(); 122 static if (OPTIONS & RedisCollectionOptions.supportIteration || OPTIONS & RedisCollectionOptions.supportPaging) { 123 static if (OPTIONS & RedisCollectionOptions.supportPaging) 124 m_db.zrem(m_allSet, id); 125 else 126 m_db.srem(m_allSet, id); 127 } 128 } 129 130 131 private string getKey(IDS ids) 132 { 133 import std.conv; 134 static if (ID_LENGTH == 1) { 135 return m_prefix[0] ~ ids.to!string ~ m_suffix; 136 } else { 137 string ret; 138 foreach (i, id; ids) ret ~= m_prefix[i] ~ id.to!string; 139 return ret ~ m_suffix; 140 } 141 } 142 } 143 144 enum RedisCollectionOptions { 145 none = 0, // Plain collection without iteration/paging support 146 supportIteration = 1<<0, // Store IDs in a set to be able to iterate and check for existence 147 supportPaging = 1<<1, // Store IDs in a sorted set, to support range based queries 148 defaults = supportIteration 149 } 150 151 152 /** Models a set of numbered hashes. 153 154 This structure is roughly equivalent to a $(D string[string][long]) and is 155 commonly used to store collections of objects, such as all users of a 156 service. For a strongly typed variant of this class, see 157 $(D RedisObjectCollection). 158 159 See_also: $(D RedisObjectCollection) 160 */ 161 template RedisHashCollection(RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 162 { 163 alias RedisHashCollection = RedisCollection!(RedisHash, OPTIONS, ID_LENGTH); 164 } 165 166 167 /** Models a strongly typed set of numbered hashes. 168 169 This structure is roughly equivalent of a $(D T[long]). 170 171 See_also: $(D RedisHashCollection) 172 */ 173 template RedisObjectCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 174 { 175 alias RedisObjectCollection = RedisCollection!(RedisObject!T, OPTIONS, ID_LENGTH); 176 } 177 178 /// 179 unittest { 180 struct User { 181 string name; 182 string email; 183 int age; 184 string password; 185 } 186 187 void test() 188 { 189 auto db = connectRedis("127.0.0.1").getDatabase(0); 190 db.deleteAll(); 191 192 auto users = RedisObjectCollection!User(db, "users"); 193 assert(users.add(User("Tom", "tom@example.com", 42, "secret")) == 0); 194 assert(users.add(User("Peter", "peter@example.com", 42, "secret")) == 1); 195 196 auto peter = users[1]; 197 assert(peter.name == "Peter"); 198 } 199 } 200 201 202 /** Models a single strongly typed object. 203 204 This structure is rougly equivalent to a value of type $(D T). The 205 underlying data is represented as a Redis hash. This means that only 206 primitive fields are supported for $(D T). 207 */ 208 struct RedisObject(T) { 209 private { 210 RedisHash!string m_hash; 211 } 212 213 this(RedisDatabase db, string key) 214 { 215 m_hash = RedisHash!string(db, key); 216 } 217 218 this(RedisHash!string hash) 219 { 220 m_hash = hash; 221 } 222 223 @property T get() 224 { 225 T ret; 226 auto repl = m_hash.database.hmget(m_hash.key, keys); 227 foreach (i, F; typeof(ret.tupleof)) { 228 assert(!repl.empty); 229 __traits(getMember, ret, keys[i]) = repl.front.fromRedis!F; 230 repl.popFront(); 231 } 232 assert(repl.empty); 233 return ret; 234 } 235 236 @property bool exists() { return m_hash.value.exists(); } 237 238 alias get this; 239 240 void remove() { m_hash.remove(); } 241 242 void opAssign(T val) 243 { 244 m_hash.database.hmset(m_hash.key, toTuple(toKeysAndValues(val)).expand); 245 } 246 247 mixin(fields()); 248 249 static private string fields() 250 { 251 string ret; 252 foreach (name; keys) { 253 ret ~= "@property auto "~name~"() { return RedisObjectField!(typeof(T."~name~"))(m_hash, \""~name~"\"); }\n"; 254 ret ~= "@property void "~name~"(typeof(T."~name~") val) { this."~name~".opAssign(val); }\n"; 255 } 256 return ret; 257 } 258 259 /*@property auto opDispatch(string name)() //if (is(typeof(getMember, T.init, name))) 260 { 261 return RedisObjectField!(typeof(__traits(getMember, T.init, name)))(m_hash, name); 262 }*/ 263 264 private static string[T.tupleof.length*2] toKeysAndValues(T val) 265 { 266 string[T.tupleof.length*2] ret; 267 enum keys = fieldNames!T; 268 foreach (i, m; val.tupleof) { 269 ret[i*2+0] = keys[i]; 270 ret[i*2+1] = m.toRedis(); 271 } 272 return ret; 273 } 274 275 private enum keys = fieldNames!T; 276 } 277 278 struct RedisObjectField(T) { 279 private { 280 RedisHash!string m_hash; 281 string m_field; 282 } 283 284 this(RedisHash!string hash, string field) 285 { 286 m_hash = hash; 287 m_field = field; 288 } 289 290 @property T get() { return m_hash.database.hget!string(m_hash.key, m_field).fromRedis!T; } 291 292 alias get this; 293 294 void opAssign(T val) { m_hash.database.hset(m_hash.key, m_field, val.toRedis); } 295 296 void opUnary(string op)() if(op == "++") { m_hash.database.hincr(m_hash.key, m_field, 1); } 297 void opUnary(string op)() if(op == "--") { m_hash.database.hincr(m_hash.key, m_field, -1); } 298 299 void opOpAssign(string op)(long val) if (op == "+") { m_hash.database.hincr(m_hash.key, m_field, val); } 300 void opOpAssign(string op)(long val) if (op == "-") { m_hash.database.hincr(m_hash.key, m_field, -val); } 301 void opOpAssign(string op)(double val) if (op == "+") { m_hash.database.hincr(m_hash.key, m_field, val); } 302 void opOpAssign(string op)(double val) if (op == "-") { m_hash.database.hincr(m_hash.key, m_field, -val); } 303 } 304 305 306 /** Models a strongly typed numbered set of values. 307 308 309 */ 310 template RedisSetCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 311 { 312 alias RedisSetCollection = RedisCollection!(RedisSet!T, OPTIONS, ID_LENGTH); 313 } 314 315 /// 316 unittest { 317 void test() 318 { 319 auto db = connectRedis("127.0.0.1").getDatabase(0); 320 auto user_groups = RedisSetCollection!(string, RedisCollectionOptions.none)(db, "user_groups"); 321 322 // add some groups for user with ID 0 323 user_groups[0].insert("cooking"); 324 user_groups[0].insert("hiking"); 325 // add some groups for user with ID 1 326 user_groups[1].insert("coding"); 327 328 assert(user_groups[0].contains("hiking")); 329 assert(!user_groups[0].contains("coding")); 330 assert(user_groups[1].contains("coding")); 331 332 user_groups[0].remove("hiking"); 333 assert(!user_groups[0].contains("hiking")); 334 } 335 } 336 337 338 /** Models a strongly typed numbered set of values. 339 340 341 */ 342 template RedisListCollection(T, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 343 { 344 alias RedisListCollection = RedisCollection!(RedisList!T, OPTIONS, ID_LENGTH); 345 } 346 347 348 /** Models a strongly typed numbered set of values. 349 350 351 */ 352 template RedisStringCollection(T = string, RedisCollectionOptions OPTIONS = RedisCollectionOptions.defaults, size_t ID_LENGTH = 1) 353 { 354 alias RedisStringCollection = RedisCollection!(RedisString!T, OPTIONS, ID_LENGTH); 355 } 356 357 358 // TODO: support distributed locking 359 struct RedisLock { 360 private { 361 RedisDatabase m_db; 362 string m_key; 363 string m_scriptSHA; 364 } 365 366 this(RedisDatabase db, string lock_key) 367 { 368 m_db = db; 369 m_key = lock_key; 370 m_scriptSHA = m_db.scriptLoad( 371 `if redis.call("get",KEYS[1]) == ARGV[1] then 372 return redis.call("del",KEYS[1]) 373 else 374 return 0 375 end`); 376 } 377 378 void performLocked(scope void delegate() del) 379 { 380 import std.random; 381 import vibe.core.core; 382 import vibe.data.bson; 383 384 auto lockval = BsonObjectID.generate(); 385 while (!m_db.setNX(m_key, cast(ubyte[])lockval, 30.seconds)) 386 sleep(uniform(1, 50).msecs); 387 388 scope (exit) m_db.evalSHA!(string, ubyte[])(m_scriptSHA, [m_key], cast(ubyte[])lockval); 389 390 del(); 391 } 392 } 393 394 395 // utility structure, temporarily placed here 396 struct JsonEncoded(T) { 397 import vibe.data.json; 398 T value; 399 400 alias value this; 401 402 static JsonEncoded fromString(string str) { return JsonEncoded(deserializeJson!T(str)); } 403 string toString() { return serializeToJsonString(value); } 404 405 static assert(isStringSerializable!JsonEncoded); 406 } 407 JsonEncoded!T jsonEncoded(T)(T value) { return JsonEncoded!T(value); } 408 409 410 // utility structure, temporarily placed here 411 struct LazyString(T...) { 412 private { 413 T m_values; 414 } 415 416 this(T values) { m_values = values; } 417 418 void toString(void delegate(string) sink) 419 { 420 foreach (v; m_values) 421 dst.formattedWrite("%s", v); 422 } 423 } 424 425 426 /** 427 Strips all non-Redis fields from a struct. 428 429 The returned struct will contain only fiels that can be converted using 430 $(D toRedis) and that have names different than "id" or "_id". 431 432 To reconstruct the full struct type, use the $(D RedisStripped.unstrip) 433 method. 434 */ 435 RedisStripped!(T, strip_id) redisStrip(bool strip_id = true, T)(in T val) { return RedisStripped!(T, strip_id)(val); } 436 437 /** 438 Represents the stripped type of a struct. 439 440 Strips all fields that cannot be directly stored as values in the Redis 441 database. By default, any field named `id` or `_id` is also stripped. Set 442 the `strip_id` parameter to `false` to keep those fields. 443 444 See_also: $(D redisStrip) 445 */ 446 struct RedisStripped(T, bool strip_id = true) { 447 import std.traits : Select, select; 448 import std.typetuple; 449 450 //pragma(msg, membersString!()); 451 mixin(membersString()); 452 453 alias StrippedMembers = FilterToType!(Select!(strip_id, isNonRedisTypeOrID, isNonRedisType), T.tupleof); 454 alias UnstrippedMembers = FilterToType!(Select!(strip_id, isRedisTypeAndNotID, isRedisType), T.tupleof); 455 alias strippedMemberIndices = indicesOf!(Select!(strip_id, isNonRedisTypeOrID, isNonRedisType), T.tupleof); 456 alias unstrippedMemberIndices = indicesOf!(Select!(strip_id, isRedisTypeAndNotID, isRedisType), T.tupleof); 457 458 this(in T src) { foreach (i, idx; unstrippedMemberIndices) this.tupleof[i] = src.tupleof[idx]; } 459 460 /** Reconstructs the full (unstripped) struct value. 461 462 The parameters for this method are all stripped fields in the order in 463 which they appear in the original struct definition. 464 */ 465 T unstrip(StrippedMembers stripped_members) { 466 T ret; 467 populateRedisFields(ret, this.tupleof); 468 populateNonRedisFields(ret, stripped_members); 469 return ret; 470 } 471 472 private void populateRedisFields(ref T dst, UnstrippedMembers values) 473 { 474 foreach (i, v; values) 475 dst.tupleof[unstrippedMemberIndices[i]] = v; 476 } 477 478 private void populateNonRedisFields(ref T dst, StrippedMembers values) 479 { 480 foreach (i, v; values) 481 dst.tupleof[strippedMemberIndices[i]] = v; 482 } 483 484 485 /*pragma(msg, T); 486 pragma(msg, "stripped: "~StrippedMembers.stringof~" - "~strippedMemberIndices.stringof); 487 pragma(msg, "unstripped: "~UnstrippedMembers.stringof~" - "~unstrippedMemberIndices.stringof);*/ 488 489 private static string membersString() 490 { 491 string ret; 492 foreach (idx; unstrippedMemberIndices) { 493 enum name = __traits(identifier, T.tupleof[idx]); 494 ret ~= "typeof(T."~name~") "~name~";\n"; 495 } 496 return ret; 497 } 498 } 499 500 unittest { 501 static struct S1 { int id; string field; string[] array; } 502 auto s1 = S1(42, "hello", ["world"]); 503 auto s1s = redisStrip(s1); 504 static assert(!is(typeof(s1s.id))); 505 static assert(is(typeof(s1s.field))); 506 static assert(!is(typeof(s1s.array))); 507 assert(s1s.field == "hello"); 508 auto s1u = s1s.unstrip(42, ["world"]); 509 assert(s1u == s1); 510 } 511 512 private template indicesOf(alias PRED, T...) 513 { 514 import std.typetuple; 515 template impl(size_t i) { 516 static if (i < T.length) { 517 static if (PRED!(T[i])) alias impl = TypeTuple!(i, impl!(i+1)); 518 else alias impl = impl!(i+1); 519 } else alias impl = TypeTuple!(); 520 } 521 alias indicesOf = impl!0; 522 } 523 private template FilterToType(alias PRED, T...) { 524 import std.typetuple; 525 template impl(size_t i) { 526 static if (i < T.length) { 527 static if (PRED!(T[i])) alias impl = TypeTuple!(typeof(T[i]), impl!(i+1)); 528 else alias impl = impl!(i+1); 529 } else alias impl = TypeTuple!(); 530 } 531 alias FilterToType = impl!0; 532 } 533 private template isRedisType(alias F) { enum isRedisType = is(typeof(&toRedis!(typeof(F)))); } 534 private template isNonRedisType(alias F) { enum isNonRedisType = !isRedisType!F; } 535 static assert(isRedisType!(int.init) && isRedisType!(string.init)); 536 static assert(!isRedisType!((float[]).init)); 537 538 private template isRedisTypeAndNotID(alias F) { import std.algorithm; enum isRedisTypeAndNotID = !__traits(identifier, F).among("_id", "id") && isRedisType!F; } 539 private template isNonRedisTypeOrID(alias F) { enum isNonRedisTypeOrID = !isRedisTypeAndNotID!F; } 540 static assert(isRedisTypeAndNotID!(int.init) && isRedisTypeAndNotID!(string.init)); 541 542 private auto toTuple(size_t N, T)(T[N] values) 543 { 544 import std.typecons; 545 import std.typetuple; 546 template impl(size_t i) { 547 static if (i < N) alias impl = TypeTuple!(T, impl!(i+1)); 548 else alias impl = TypeTuple!(); 549 } 550 Tuple!(impl!0) ret; 551 foreach (i, T; impl!0) ret[i] = values[i]; 552 return ret; 553 } 554 555 private template fieldNames(T) 556 { 557 import std.typetuple; 558 template impl(size_t i) { 559 static if (i < T.tupleof.length) 560 alias impl = TypeTuple!(__traits(identifier, T.tupleof[i]), impl!(i+1)); 561 else alias impl = TypeTuple!(); 562 } 563 enum string[T.tupleof.length] fieldNames = [impl!0]; 564 } 565 566 unittest { 567 static struct Test { int a; float b; void method() {} Test[] c; void opAssign(Test) {}; ~this() {} } 568 static assert(fieldNames!Test[] == ["a", "b", "c"]); 569 } 570 571 private template Replicate(T, size_t L) 572 { 573 import std.typetuple; 574 static if (L > 0) { 575 alias Replicate = TypeTuple!(T, Replicate!(T, L-1)); 576 } else alias Replicate = TypeTuple!(); 577 }