From 276cf3c8d893e5502c52d522182f4d4accb795fc Mon Sep 17 00:00:00 2001 From: Lee Smet Date: Wed, 23 Apr 2025 13:53:10 +0200 Subject: [PATCH] Add alternative DB interface and implementation Signed-off-by: Lee Smet --- heromodels/Cargo.lock | 590 +++++++++++++++++++++- heromodels/Cargo.toml | 7 +- heromodels/examples/basic_user_example.rs | 131 ++++- heromodels/src/db.rs | 69 +++ heromodels/src/db/fjall.rs | 0 heromodels/src/db/hero.rs | 269 ++++++++++ heromodels/src/lib.rs | 5 +- heromodels/src/models/core/mod.rs | 2 +- heromodels/src/models/core/model.rs | 9 + heromodels/src/models/mod.rs | 2 +- heromodels/src/models/userexample/user.rs | 32 +- 11 files changed, 1099 insertions(+), 17 deletions(-) create mode 100644 heromodels/src/db.rs create mode 100644 heromodels/src/db/fjall.rs create mode 100644 heromodels/src/db/hero.rs diff --git a/heromodels/Cargo.lock b/heromodels/Cargo.lock index b60e503..7c756a5 100644 --- a/heromodels/Cargo.lock +++ b/heromodels/Cargo.lock @@ -25,19 +25,48 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "bincode" -version = "1.3.3" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" dependencies = [ + "bincode_derive", "serde", + "unty", ] +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + +[[package]] +name = "bitflags" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" + [[package]] name = "bumpalo" version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "byteview" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6236364b88b9b6d0bc181ba374cf1ab55ba3ef97a1cb6f8cddad48a273767fb5" + [[package]] name = "cc" version = "1.2.19" @@ -68,19 +97,174 @@ dependencies = [ "windows-link", ] +[[package]] +name = "compare" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea0095f6103c2a8b44acd6fd15960c801dafebf02e21940360833e0673f48ba7" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-skiplist" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df29de440c58ca2cc6e587ec3d22347551a32435fbde9d2bff64e78a9ffa151b" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "double-ended-peekable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0d05e1c0dbad51b52c38bda7adceef61b9efc2baf04acfe8726a8c4630a6f57" + +[[package]] +name = "enum_dispatch" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" +dependencies = [ + "once_cell", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fjall" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958511f67d1f80e6bff9ffac05c626bb340d4602ca6ea5617d9901c218c894f0" +dependencies = [ + "byteorder", + "byteview", + "dashmap", + "log", + "lsm-tree", + "path-absolutize", + "std-semaphore", + "tempfile", + "xxhash-rust", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "guardian" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e2ac29387b1aa07a1e448f7bb4f35b500787971e965b02842b900afa5c8f6f" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + [[package]] name = "heromodels" version = "0.1.0" dependencies = [ "bincode", "chrono", + "fjall", + "ourdb", "serde", + "tst", ] [[package]] @@ -107,6 +291,15 @@ dependencies = [ "cc", ] +[[package]] +name = "interval-heap" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11274e5e8e89b8607cfedc2910b6626e998779b48a019151c7604d0adcb86ac6" +dependencies = [ + "compare", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -123,12 +316,58 @@ version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lsm-tree" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87d58bdef2dcbf50fce9f343265bdbd7fb08a458d241eb837ce426be22d674b4" +dependencies = [ + "byteorder", + "crossbeam-skiplist", + "double-ended-peekable", + "enum_dispatch", + "guardian", + "interval-heap", + "log", + "lz4_flex", + "path-absolutize", + "quick_cache", + "rustc-hash", + "self_cell", + "tempfile", + "value-log", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" + [[package]] name = "num-traits" version = "0.2.19" @@ -144,6 +383,56 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "ourdb" +version = "0.1.0" +dependencies = [ + "crc32fast", + "log", + "rand", + "thiserror", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -153,6 +442,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick_cache" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "287e56aac5a2b4fb25a6fb050961d157635924c8696305a5c937a76f29841a0f" +dependencies = [ + "equivalent", + "hashbrown 0.15.2", +] + [[package]] name = "quote" version = "1.0.40" @@ -162,12 +461,88 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.15", +] + +[[package]] +name = "redox_syscall" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustix" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustversion" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "self_cell" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f7d95a54511e0c7be3f51e8867aa8cf35148d7b9445d44de2f943e2b206e749" + [[package]] name = "serde" version = "1.0.219" @@ -194,6 +569,18 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "smallvec" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" + +[[package]] +name = "std-semaphore" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" + [[package]] name = "syn" version = "2.0.100" @@ -205,12 +592,103 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tempfile" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +dependencies = [ + "fastrand", + "getrandom 0.3.2", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tst" +version = "0.1.0" +dependencies = [ + "ourdb", + "thiserror", +] + [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + +[[package]] +name = "value-log" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62fc7c4ce161f049607ecea654dca3f2d727da5371ae85e2e4f14ce2b98ed67c" +dependencies = [ + "byteorder", + "byteview", + "interval-heap", + "log", + "path-absolutize", + "rustc-hash", + "tempfile", + "varint-rs", + "xxhash-rust", +] + +[[package]] +name = "varint-rs" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f54a172d0620933a27a4360d3db3e2ae0dd6cceae9730751a036bbf182c4b23" + +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -327,3 +805,111 @@ checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" dependencies = [ "windows-link", ] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/heromodels/Cargo.toml b/heromodels/Cargo.toml index 7258a64..6f04222 100644 --- a/heromodels/Cargo.toml +++ b/heromodels/Cargo.toml @@ -7,5 +7,8 @@ authors = ["Your Name "] [dependencies] serde = { version = "1.0", features = ["derive"] } -bincode = "1.3" -chrono = { version = "0.4", features = ["serde"] } \ No newline at end of file +bincode = { version = "2", features = ["serde"] } +chrono = { version = "0.4", features = ["serde"] } +fjall = "2.9.0" +ourdb = { path = "../ourdb" } +tst = { path = "../tst" } diff --git a/heromodels/examples/basic_user_example.rs b/heromodels/examples/basic_user_example.rs index 03fd28a..475502f 100644 --- a/heromodels/examples/basic_user_example.rs +++ b/heromodels/examples/basic_user_example.rs @@ -1,28 +1,141 @@ -use heromodels::models::{Model, Comment, User}; +use heromodels::db::{Collection, Db}; +use heromodels::models::userexample::user::{IsActive, UserName}; +use heromodels::models::{Comment, Model, User}; fn main() { + let index_db = tst::TST::new("/tmp/ourdb/tst", true).expect("can create index DB"); + let data_db = ourdb::OurDB::new(ourdb::OurDBConfig { + incremental_mode: false, + path: "/tmp/ourdb/ourdb".into(), + file_size: None, + keysize: None, + reset: Some(true), + }) + .expect("can create data DB"); + let db = heromodels::db::hero::OurDB::new(index_db, data_db); + println!("Hero Models - Basic Usage Example"); println!("================================"); - + // Create a new user using the fluent interface let user = User::new(1) .username("johndoe") .email("john.doe@example.com") .full_name("John Doe") + .is_active(false) .build(); - + let user2 = User::new(2) + .username("janesmith") + .email("jane.smith@example.com") + .full_name("Jane Smith") + .is_active(true) + .build(); + let user3 = User::new(3) + .username("willism") + .email("willis.masters@example.com") + .full_name("Willis Masters") + .is_active(true) + .build(); + let user4 = User::new(4) + .username("carrols") + .email("carrol.smith@example.com") + .full_name("Carrol Smith") + .is_active(false) + .build(); + + db.collection() + .expect("can open user collection") + .set(&user) + .expect("can set user"); + db.collection() + .expect("can open user collection") + .set(&user2) + .expect("can set user"); + db.collection() + .expect("can open user collection") + .set(&user3) + .expect("can set user"); + db.collection() + .expect("can open user collection") + .set(&user4) + .expect("can set user"); + + // Perform an indexed lookup on the Username + let stored_users = db + .collection::() + .expect("can open user collection") + .get::("johndoe") + .expect("can load stored user"); + + assert_eq!(stored_users.len(), 1); + let stored_user = &stored_users[0]; + + assert_eq!(user.username, stored_user.username); + assert_eq!(user.email, stored_user.email); + assert_eq!(user.is_active, stored_user.is_active); + assert_eq!(user.full_name, stored_user.full_name); + + // Load all active users using the IsActive field index + // TODO: expand Index type so it defines the type of the key + let key = true.to_string(); + let active_users = db + .collection::() + .expect("can open user collection") + .get::(&key) + .expect("can load stored users"); + // We should have 2 active users + assert_eq!(active_users.len(), 2); + + // Now remove a user + db.collection::() + .expect("can open user collection") + .delete_by_id(active_users[0].get_id()) + .expect("can delete existing user"); + + // Load the active users again, should be 1 left + let active_users = db + .collection::() + .expect("can open user collection") + .get::(&key) + .expect("can load stored users"); + assert_eq!(active_users.len(), 1); + // And verify we still have 2 inactive users + let key = false.to_string(); + let inactive_users = db + .collection::() + .expect("can open user collection") + .get::(&key) + .expect("can load stored users"); + assert_eq!(inactive_users.len(), 2); + println!("Created user: {:?}", user); println!("User ID: {}", user.get_id()); println!("User DB Prefix: {}", User::db_prefix()); - + // Create a comment for the user - let comment = Comment::new(1) - .user_id(2) // commenter's user ID + let comment = Comment::new(5) + .user_id(1) // commenter's user ID .content("This is a comment on the user") .build(); - + + db.collection() + .expect("can open commen collection") + .set(&comment) + .expect("can set comment"); + + let stored_comment = db + .collection::() + .expect("can open comment collection") + .get_by_id(5) + .expect("can load stored comment"); + + assert!(stored_comment.is_some()); + let stored_comment = stored_comment.unwrap(); + + assert_eq!(comment.get_id(), stored_comment.get_id()); + assert_eq!(comment.content, stored_comment.content); + println!("\nCreated comment: {:?}", comment); println!("Comment ID: {}", comment.get_id()); println!("Comment DB Prefix: {}", Comment::db_prefix()); - -} \ No newline at end of file +} diff --git a/heromodels/src/db.rs b/heromodels/src/db.rs new file mode 100644 index 0000000..5f74393 --- /dev/null +++ b/heromodels/src/db.rs @@ -0,0 +1,69 @@ +use crate::models::{Index, Model}; +use serde::{Deserialize, Serialize}; + +pub mod fjall; +pub mod hero; + +pub trait Db { + /// Error type returned by database operations. + type Error: std::fmt::Debug; + + /// Open the collection for a specific model. This method must create the collection if it does not exist. + fn collection(&self) -> Result, Error>; +} + +/// A collection stores a specific model under a specific key +pub trait Collection +where + K: Serialize, + V: Serialize + for<'a> Deserialize<'a>, +{ + /// Error type for database operations + type Error: std::fmt::Debug; + + /// Get all items where the given index field is equal to key. + fn get(&self, key: K) -> Result, Error> + where + I: Index; + + /// Get an object from its ID. This does not use an index lookup + fn get_by_id(&self, id: u32) -> Result, Error>; + + /// Store an item in the DB. + fn set(&self, value: &V) -> Result<(), Error>; + + /// Delete all items from the db with a given index. + fn delete(&self, key: K) -> Result<(), Error> + where + I: Index; + + /// Delete an object with a given ID + fn delete_by_id(&self, id: u32) -> Result<(), Error>; + + /// Get all objects from the colelction + fn get_all(&self) -> Result, Error>; +} + +/// Errors returned by the DB implementation +#[derive(Debug)] +pub enum Error { + /// Error in the underlying database + DB(E), + /// Error decoding a stored model + Decode(bincode::error::DecodeError), + /// Error encoding a model for storage + Encode(bincode::error::EncodeError), +} + +impl From for Error { + fn from(value: bincode::error::DecodeError) -> Self { + Error::Decode(value) + } +} + +impl From for Error { + fn from(value: bincode::error::EncodeError) -> Self { + Error::Encode(value) + } +} + diff --git a/heromodels/src/db/fjall.rs b/heromodels/src/db/fjall.rs new file mode 100644 index 0000000..e69de29 diff --git a/heromodels/src/db/hero.rs b/heromodels/src/db/hero.rs new file mode 100644 index 0000000..e918176 --- /dev/null +++ b/heromodels/src/db/hero.rs @@ -0,0 +1,269 @@ +use ourdb::OurDBSetArgs; +use serde::Deserialize; + +use crate::models::{Index, Model}; + +use std::{ + collections::HashSet, + sync::{Arc, Mutex}, +}; + +const BINCODE_CONFIG: bincode::config::Configuration = bincode::config::standard(); + +#[derive(Clone)] +pub struct OurDB { + index: Arc>, + data: Arc>, +} + +impl OurDB { + /// Create a new instance of ourdb + pub fn new(index_db: tst::TST, data_db: ourdb::OurDB) -> Self { + Self { + index: Arc::new(Mutex::new(index_db)), + data: Arc::new(Mutex::new(data_db)), + } + } +} + +impl super::Db for OurDB { + type Error = tst::Error; + + fn collection( + &self, + ) -> Result, super::Error> { + Ok(self.clone()) + } +} + +impl super::Collection<&str, M> for OurDB +where + M: Model, +{ + type Error = tst::Error; + + fn get(&self, key: &str) -> Result, super::Error> + where + I: Index, + { + let mut index_db = self.index.lock().expect("can lock index DB"); + let index_key = Self::index_key(M::db_prefix(), I::key(), key); + + let Some(object_ids) = Self::get_tst_value::>(&mut index_db, &index_key)? + else { + // If the index is not found just return an empty vector, no items present + return Ok(vec![]); + }; + + let mut data_db = self.data.lock().expect("can lock data DB"); + object_ids + .into_iter() + .map(|obj_id| -> Result { + let raw_obj = data_db.get(obj_id)?; + let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw_obj, BINCODE_CONFIG)?; + Ok(obj) + }) + .collect() + } + + fn get_by_id(&self, id: u32) -> Result, super::Error> { + let mut data_db = self.data.lock().expect("can lock data db"); + Self::get_ourdb_value(&mut data_db, id) + } + + fn set(&self, value: &M) -> Result<(), super::Error> { + // Before inserting the new object, check if an object with this ID already exists. If it does, we potentially need to update indices. + let mut data_db = self.data.lock().expect("can lock data DB"); + let old_obj: Option = Self::get_ourdb_value(&mut data_db, value.get_id())?; + let (indices_to_delete, indices_to_add) = if let Some(old_obj) = old_obj { + let mut indices_to_delete = vec![]; + let mut indices_to_add = vec![]; + let old_indices = old_obj.db_keys(); + let new_indices = value.db_keys(); + for old_index in old_indices { + for new_index in &new_indices { + if old_index.name == new_index.name { + if old_index.value != new_index.value { + // different value now, remove index + indices_to_delete.push(old_index); + // and later add the new one + indices_to_add.push(new_index.clone()); + break; + } + } + } + } + // NOTE: we assume here that the index keys are stable, i.e. new index fields don't appear + // and existing ones don't dissapear + (indices_to_delete, indices_to_add) + } else { + (vec![], value.db_keys()) + }; + + let mut index_db = self.index.lock().expect("can lock index db"); + // First delete old indices which need to change + for old_index in indices_to_delete { + let key = Self::index_key(M::db_prefix(), old_index.name, &old_index.value); + let raw_ids = index_db.get(&key)?; + let (mut ids, _): (HashSet, _) = + bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?; + ids.remove(&value.get_id()); + if ids.is_empty() { + // This was the last ID with this index value, remove index entirely + index_db.delete(&key)?; + } else { + // There are still objects left with this index value, write back updated set + let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?; + index_db.set(&key, raw_ids)?; + } + } + + // set or update the object + let v = bincode::serde::encode_to_vec(value, BINCODE_CONFIG)?; + let id = value.get_id(); + data_db.set(OurDBSetArgs { + id: Some(id), + data: &v, + })?; + + // Now add the new indices + for index_key in indices_to_add { + let key = Self::index_key(M::db_prefix(), index_key.name, &index_key.value); + // Load the existing id set for the index or create a new set + let mut existing_ids = + Self::get_tst_value::>(&mut index_db, &key)?.unwrap_or_default(); + existing_ids.insert(id); + let encoded_ids = bincode::serde::encode_to_vec(existing_ids, BINCODE_CONFIG)?; + index_db.set(&key, encoded_ids)?; + } + + Ok(()) + } + + fn delete(&self, key: &str) -> Result<(), super::Error> + where + I: Index, + { + let mut index_db = self.index.lock().expect("can lock index db"); + let key = Self::index_key(M::db_prefix(), I::key(), key); + let raw_obj_ids = index_db.get(&key)?; + let (obj_ids, _): (HashSet, _) = + bincode::serde::decode_from_slice(&raw_obj_ids, BINCODE_CONFIG)?; + + let mut data_db = self.data.lock().expect("can lock data DB"); + for obj_id in obj_ids { + // Load object + let raw = data_db.get(obj_id)?; + let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?; + + // Delete indices + for index in obj.db_keys() { + let key = Self::index_key(M::db_prefix(), index.name, &index.value); + let raw_ids = index_db.get(&key)?; + let (mut ids, _): (HashSet, _) = + bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?; + ids.remove(&obj_id); + if ids.is_empty() { + // This was the last ID with this index value, remove index entirely + index_db.delete(&key)?; + } else { + // There are still objects left with this index value, write back updated set + let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?; + index_db.set(&key, raw_ids)?; + } + } + + // Delete object + data_db.delete(obj_id)?; + } + + Ok(()) + } + + fn delete_by_id(&self, id: u32) -> Result<(), super::Error> { + let mut data_db = self.data.lock().expect("can lock data DB"); + let raw = data_db.get(id)?; + + // First load the object so we can delete the indices + // Delete indices before the actual object, so we don't leave dangling references to a deleted + // object ID in case something goes wrong. + let (obj, _): (M, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?; + let mut index_db = self.index.lock().expect("can lock index DB"); + for index_key in obj.db_keys() { + let key = Self::index_key(M::db_prefix(), index_key.name, &index_key.value); + let raw_ids = index_db.get(&key)?; + let (mut ids, _): (HashSet, _) = + bincode::serde::decode_from_slice(&raw_ids, BINCODE_CONFIG)?; + ids.remove(&obj.get_id()); + if ids.is_empty() { + // This was the last ID with this index value, remove index entirely + index_db.delete(&key)?; + } else { + // There are still objects left with this index value, write back updated set + let raw_ids = bincode::serde::encode_to_vec(ids, BINCODE_CONFIG)?; + index_db.set(&key, raw_ids)?; + } + } + + // Finally delete the object itself + Ok(data_db.delete(id)?) + } + + fn get_all(&self) -> Result, super::Error> { + todo!("OurDB doesn't have a list all method yet") + } +} + +impl OurDB { + /// Build the key used for an indexed field of a collection. + fn index_key(collection: &str, index: &str, value: &str) -> String { + format!("{collection}::{index}::{value}") + } + + /// Wrapper to load values from ourdb and transform a not found error in to Ok(None) + fn get_ourdb_value( + data: &mut ourdb::OurDB, + id: u32, + ) -> Result, super::Error> + where + V: for<'de> Deserialize<'de>, + { + match data.get(id) { + Ok(raw) => { + let (obj, _): (V, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?; + Ok(Some(obj)) + } + Err(ourdb::Error::NotFound(_)) => Ok(None), + Err(e) => Err(e.into()), + } + } + + fn get_tst_value( + index: &mut tst::TST, + key: &str, + ) -> Result, super::Error> + where + V: for<'de> Deserialize<'de>, + { + match index.get(key) { + Ok(raw) => { + let (obj, _): (V, _) = bincode::serde::decode_from_slice(&raw, BINCODE_CONFIG)?; + Ok(Some(obj)) + } + Err(tst::Error::KeyNotFound(_) | tst::Error::PrefixNotFound(_)) => Ok(None), + Err(e) => Err(e.into()), + } + } +} + +impl From for super::Error { + fn from(value: tst::Error) -> Self { + super::Error::DB(value) + } +} + +impl From for super::Error { + fn from(value: ourdb::Error) -> Self { + super::Error::DB(tst::Error::OurDB(value)) + } +} diff --git a/heromodels/src/lib.rs b/heromodels/src/lib.rs index eecd6d0..58f4ea7 100644 --- a/heromodels/src/lib.rs +++ b/heromodels/src/lib.rs @@ -1,2 +1,5 @@ // Export the models module -pub mod models; \ No newline at end of file +pub mod models; + +/// Database implementations +pub mod db; diff --git a/heromodels/src/models/core/mod.rs b/heromodels/src/models/core/mod.rs index 6160f5d..2524f23 100644 --- a/heromodels/src/models/core/mod.rs +++ b/heromodels/src/models/core/mod.rs @@ -3,5 +3,5 @@ pub mod model; pub mod comment; // Re-export key types for convenience -pub use model::{Model, BaseModelData, IndexKey, IndexKeyBuilder}; +pub use model::{Model, BaseModelData, IndexKey, IndexKeyBuilder, Index}; pub use comment::Comment; \ No newline at end of file diff --git a/heromodels/src/models/core/model.rs b/heromodels/src/models/core/model.rs index e289dc6..02e5583 100644 --- a/heromodels/src/models/core/model.rs +++ b/heromodels/src/models/core/model.rs @@ -73,6 +73,15 @@ pub trait Model: Debug + Clone + Serialize + for<'de> Deserialize<'de> + Send + } } +/// An identifier for an index in the DB +pub trait Index { + /// The model for which this is an index in the database + type Model: Model; + + /// The key of this index + fn key() -> &'static str; +} + /// Base struct that all models should include #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BaseModelData { diff --git a/heromodels/src/models/mod.rs b/heromodels/src/models/mod.rs index 478295a..a12ce27 100644 --- a/heromodels/src/models/mod.rs +++ b/heromodels/src/models/mod.rs @@ -3,6 +3,6 @@ pub mod core; pub mod userexample; // Re-export key types for convenience -pub use core::model::{Model, BaseModelData, IndexKey}; +pub use core::model::{Model, BaseModelData, IndexKey, Index}; pub use core::Comment; pub use userexample::User; \ No newline at end of file diff --git a/heromodels/src/models/userexample/user.rs b/heromodels/src/models/userexample/user.rs index 140ca0d..6ee421a 100644 --- a/heromodels/src/models/userexample/user.rs +++ b/heromodels/src/models/userexample/user.rs @@ -1,5 +1,5 @@ use serde::{Deserialize, Serialize}; -use crate::models::core::model::{Model, BaseModelData, IndexKey}; +use crate::models::core::model::{Model, BaseModelData, IndexKey, Index}; /// Represents a user in the system #[derive(Debug, Clone, Serialize, Deserialize)] @@ -109,3 +109,33 @@ impl Model for User { ] } } + +// Marker structs for indexed fields + +pub struct UserName; +pub struct Email; +pub struct IsActive; + +impl Index for UserName { + type Model = User; + + fn key() -> &'static str { + "username" + } +} + +impl Index for Email { + type Model = User; + + fn key() -> &'static str { + "email" + } +} + +impl Index for IsActive { + type Model = User; + + fn key() -> &'static str { + "is_active" + } +} \ No newline at end of file