herocode/db
11
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

add heroledger models

Browse Source
This commit is contained in:
Timur Gordon
2025-08-08 09:46:30 +02:00
parent 0cffda37a7
commit 6727c7498d
58 changed files with 507 additions and 10894 deletions
Download Patch File Download Diff File Expand all files Collapse all files

462
Cargo.lock generated
View File

@@ -31,15 +31,6 @@ dependencies = [
"zerocopy",
]
[[package]]
name = "aho-corasick"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
"memchr",
]
[[package]]
name = "android-tzdata"
version = "0.1.1"
@@ -55,18 +46,6 @@ dependencies = [
"libc",
]
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstyle"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd"
[[package]]
name = "arrayvec"
version = "0.7.6"
@@ -102,7 +81,7 @@ dependencies = [
"miniz_oxide",
"object",
"rustc-demangle",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@@ -164,12 +143,6 @@ version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.2.31"
@@ -200,58 +173,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "ciborium"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
[[package]]
name = "ciborium-ll"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "clap"
version = "4.5.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.5.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966"
dependencies = [
"anstyle",
"clap_lex",
]
[[package]]
name = "clap_lex"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
[[package]]
name = "const-random"
version = "0.1.18"
@@ -296,67 +217,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
dependencies = [
"anes",
"cast",
"ciborium",
"clap",
"criterion-plot",
"is-terminal",
"itertools",
"num-traits",
"once_cell",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.4"
@@ -384,28 +244,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "equivalent"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
version = "0.3.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad"
dependencies = [
"libc",
"windows-sys 0.60.2",
]
[[package]]
name = "ethnum"
version = "1.5.2"
@@ -424,12 +268,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55"
[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "futures-channel"
version = "0.3.31"
@@ -523,16 +361,6 @@ version = "0.31.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f"
[[package]]
name = "half"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9"
dependencies = [
"cfg-if",
"crunchy",
]
[[package]]
name = "hashbrown"
version = "0.15.4"
@@ -545,12 +373,6 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hermit-abi"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
[[package]]
name = "heromodels"
version = "0.1.0"
@@ -655,26 +477,6 @@ dependencies = [
"libc",
]
[[package]]
name = "is-terminal"
version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
]
[[package]]
name = "itertools"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.15"
@@ -758,12 +560,6 @@ version = "0.2.174"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "lock_api"
version = "0.4.13"
@@ -861,12 +657,6 @@ dependencies = [
"portable-atomic",
]
[[package]]
name = "oorandom"
version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "ordered-float"
version = "5.0.0"
@@ -881,10 +671,8 @@ name = "ourdb"
version = "0.1.0"
dependencies = [
"crc32fast",
"criterion",
"log",
"rand 0.8.5",
"tempfile",
"thiserror",
]
@@ -908,7 +696,7 @@ dependencies = [
"libc",
"redox_syscall",
"smallvec",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@@ -947,34 +735,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "plotters"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
[[package]]
name = "plotters-svg"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
dependencies = [
"plotters-backend",
]
[[package]]
name = "portable-atomic"
version = "1.11.1"
@@ -1089,17 +849,6 @@ dependencies = [
"r2d2",
]
[[package]]
name = "radixtree"
version = "0.1.0"
dependencies = [
"criterion",
"log",
"ourdb",
"tempfile",
"thiserror",
]
[[package]]
name = "rand"
version = "0.8.5"
@@ -1159,26 +908,6 @@ dependencies = [
"getrandom 0.3.3",
]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "redox_syscall"
version = "0.5.17"
@@ -1188,35 +917,6 @@ dependencies = [
"bitflags",
]
[[package]]
name = "regex"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rhai"
version = "1.22.2"
@@ -1263,19 +963,6 @@ version = "0.1.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
[[package]]
name = "rustix"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.60.2",
]
[[package]]
name = "rustversion"
version = "1.0.21"
@@ -1288,15 +975,6 @@ version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "scheduled-thread-pool"
version = "0.2.7"
@@ -1479,19 +1157,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1"
dependencies = [
"fastrand",
"getrandom 0.3.3",
"once_cell",
"rustix",
"windows-sys 0.59.0",
]
[[package]]
name = "thin-vec"
version = "0.2.14"
@@ -1527,16 +1192,6 @@ dependencies = [
"crunchy",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "tinyvec"
version = "1.9.0"
@@ -1692,16 +1347,6 @@ version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -1802,15 +1447,6 @@ dependencies = [
"web-sys",
]
[[package]]
name = "winapi-util"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "windows-core"
version = "0.61.2"
@@ -1876,7 +1512,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@@ -1885,16 +1521,7 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.3",
"windows-targets",
]
[[package]]
@@ -1903,31 +1530,14 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.0",
"windows_aarch64_msvc 0.53.0",
"windows_i686_gnu 0.53.0",
"windows_i686_gnullvm 0.53.0",
"windows_i686_msvc 0.53.0",
"windows_x86_64_gnu 0.53.0",
"windows_x86_64_gnullvm 0.53.0",
"windows_x86_64_msvc 0.53.0",
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
@@ -1936,96 +1546,48 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
[[package]]
name = "wit-bindgen-rt"
version = "0.39.0"

3
Cargo.toml
View File

@@ -3,7 +3,4 @@ members = [
"heromodels",
"heromodels_core",
"heromodels-derive",
"ourdb",
"radixtree",
"tst",
]

4
heromodels/Cargo.toml
View File

@@ -10,8 +10,8 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
bincode = { version = "2", features = ["serde"] }
chrono = { version = "0.4", features = ["serde"] }
ourdb = { path = "../ourdb" }
tst = { path = "../tst" }
ourdb = { path = "../../herolib_rust/packages/data/ourdb" }
tst = { path = "../../herolib_rust/packages/data/tst" }
heromodels-derive = { path = "../heromodels-derive" }
heromodels_core = { path = "../heromodels_core" }
rhai = { version = "1.21.0", features = [

1
heromodels/docs/prompts/v_specs_to_rust_heromodels.md
View File

@@ -52,7 +52,6 @@ pub mut:
use heromodels_core::{Model, BaseModelData, IndexKey};
use heromodels_derive::model;
use rhai::CustomType;
use rhailib_derive::RhaiApi;
use serde::{Deserialize, Serialize};
use chrono::{DateTime, Utc};
```

53
heromodels/examples/heroledger/heroledger.rhai Normal file
View File

@@ -0,0 +1,53 @@
// heroledger.rhai - Demonstration of HeroLedger models in Rhai
print("=== HeroLedger Models Demo ===");
// Create a new user
print("\n--- Creating User ---");
let new_user = new_user()
.name("Alice Johnson")
.email("alice@herocode.com")
.pubkey("0x1234567890abcdef")
.status("Active")
.save_user();
print("Created user: " + new_user.get_name());
print("User ID: " + new_user.get_id());
print("User email: " + new_user.get_email());
print("User pubkey: " + new_user.get_pubkey());
// Create a new group
print("\n--- Creating Group ---");
let new_group = new_group()
.name("HeroCode Developers")
.description("A group for HeroCode development team members")
.visibility("Public")
.save_group();
print("Created group: " + new_group.get_name());
print("Group ID: " + new_group.get_id());
print("Group description: " + new_group.get_description());
// Create a new account
print("\n--- Creating Account ---");
let new_account = new_account()
.name("Alice's Main Account")
.description("Primary account for Alice Johnson")
.currency("USD")
.save_account();
print("Created account: " + new_account.get_name());
print("Account ID: " + new_account.get_id());
print("Account currency: " + new_account.get_currency());
// Create a new DNS zone
print("\n--- Creating DNS Zone ---");
let new_dns_zone = new_dns_zone()
.name("herocode.com")
.description("Main domain for HeroCode")
.save_dns_zone();
print("Created DNS zone: " + new_dns_zone.get_name());
print("DNS zone ID: " + new_dns_zone.get_id());
print("\n=== Demo Complete ===");

50
heromodels/examples/heroledger/main.rs Normal file
View File

@@ -0,0 +1,50 @@
use heromodels_core::db::hero::OurDB;
use rhai::{Dynamic, Engine};
use heromodels::models::heroledger::rhai::register_heroledger_rhai_modules;
use std::sync::Arc;
use std::{fs, path::Path};
const CALLER_ID: &str = "example_caller";
fn main() -> Result<(), Box<dyn std::error::Error>> {
// Initialize Rhai engine
let mut engine = Engine::new();
// Initialize database with OurDB
let db_path = "temp_heroledger_db";
// Clean up previous database file if it exists
if Path::new(db_path).exists() {
fs::remove_dir_all(db_path)?;
}
let _db = Arc::new(OurDB::new(db_path, true).expect("Failed to create database"));
// Register the heroledger modules with Rhai
register_heroledger_rhai_modules(&mut engine);
let mut db_config = rhai::Map::new();
db_config.insert("DB_PATH".into(), db_path.into());
db_config.insert("CALLER_ID".into(), CALLER_ID.into());
db_config.insert("CONTEXT_ID".into(), CALLER_ID.into());
engine.set_default_tag(Dynamic::from(db_config)); // Or pass via CallFnOptions
// Load and evaluate the Rhai script
let manifest_dir = env!("CARGO_MANIFEST_DIR");
let script_path = Path::new(manifest_dir)
.join("examples")
.join("heroledger")
.join("heroledger.rhai");
println!("Script path: {}", script_path.display());
let script = fs::read_to_string(&script_path)?;
println!("--- Running HeroLedger Rhai Script ---");
match engine.eval::<()>(&script) {
Ok(_) => println!("\n--- Script executed successfully! ---"),
Err(e) => eprintln!("\n--- Script execution failed: {} ---", e),
}
// Clean up the database file
fs::remove_dir_all(db_path)?;
println!("--- Cleaned up temporary database. ---");
Ok(())
}

1
heromodels/src/models/heroledger/mod.rs
View File

@@ -7,6 +7,7 @@ pub mod dnsrecord;
pub mod secretbox;
pub mod signature;
pub mod user_kvs;
pub mod rhai;
// Re-export key types for convenience
pub use user::{User, UserStatus, UserProfile, KYCInfo, KYCStatus, SecretBox};

330
heromodels/src/models/heroledger/rhai.rs Normal file
View File

@@ -0,0 +1,330 @@
use ::rhai::plugin::*;
use ::rhai::{Array, Dynamic, Engine, EvalAltResult, Map, Module};
use std::mem;
use crate::models::heroledger::*;
// ============================================================================
// User Module
// ============================================================================
type RhaiUser = User;
#[export_module]
mod rhai_user_module {
use super::RhaiUser;
#[rhai_fn(name = "new_user", return_raw)]
pub fn new_user() -> Result<RhaiUser, Box<EvalAltResult>> {
Ok(User::new(0))
}
#[rhai_fn(name = "username", return_raw)]
pub fn set_username(
user: &mut RhaiUser,
username: String,
) -> Result<RhaiUser, Box<EvalAltResult>> {
let owned = std::mem::take(user);
*user = owned.username(username);
Ok(user.clone())
}
#[rhai_fn(name = "add_email", return_raw)]
pub fn add_email(
user: &mut RhaiUser,
email: String,
) -> Result<RhaiUser, Box<EvalAltResult>> {
let owned = std::mem::take(user);
*user = owned.add_email(email);
Ok(user.clone())
}
#[rhai_fn(name = "pubkey", return_raw)]
pub fn set_pubkey(
user: &mut RhaiUser,
pubkey: String,
) -> Result<RhaiUser, Box<EvalAltResult>> {
let owned = std::mem::take(user);
*user = owned.pubkey(pubkey);
Ok(user.clone())
}
#[rhai_fn(name = "status", return_raw)]
pub fn set_status(
user: &mut RhaiUser,
status: String,
) -> Result<RhaiUser, Box<EvalAltResult>> {
let status_enum = match status.as_str() {
"Active" => UserStatus::Active,
"Inactive" => UserStatus::Inactive,
"Suspended" => UserStatus::Suspended,
"Archived" => UserStatus::Archived,
_ => return Err(format!("Invalid user status: {}", status).into()),
};
let owned = std::mem::take(user);
*user = owned.status(status_enum);
Ok(user.clone())
}
#[rhai_fn(name = "save_user", return_raw)]
pub fn save_user(user: &mut RhaiUser) -> Result<RhaiUser, Box<EvalAltResult>> {
// This would integrate with the database save functionality
// For now, just return the user as-is
Ok(user.clone())
}
// Getters
#[rhai_fn(name = "get_id")]
pub fn get_id(user: &mut RhaiUser) -> i64 {
user.base_data.id as i64
}
#[rhai_fn(name = "get_username")]
pub fn get_username(user: &mut RhaiUser) -> String {
user.username.clone().unwrap_or_else(|| String::new())
}
#[rhai_fn(name = "get_email")]
pub fn get_email(user: &mut RhaiUser) -> String {
if let Some(first_email) = user.email.first() {
first_email.clone()
} else {
String::new()
}
}
#[rhai_fn(name = "get_pubkey")]
pub fn get_pubkey(user: &mut RhaiUser) -> String {
user.pubkey.clone().unwrap_or_else(|| String::new())
}
}
// ============================================================================
// Group Module
// ============================================================================
type RhaiGroup = Group;
#[export_module]
mod rhai_group_module {
use super::RhaiGroup;
#[rhai_fn(name = "new_group", return_raw)]
pub fn new_group() -> Result<RhaiGroup, Box<EvalAltResult>> {
Ok(Group::new(0))
}
#[rhai_fn(name = "name", return_raw)]
pub fn set_name(
group: &mut RhaiGroup,
name: String,
) -> Result<RhaiGroup, Box<EvalAltResult>> {
let owned = std::mem::take(group);
*group = owned.name(name);
Ok(group.clone())
}
#[rhai_fn(name = "description", return_raw)]
pub fn set_description(
group: &mut RhaiGroup,
description: String,
) -> Result<RhaiGroup, Box<EvalAltResult>> {
let owned = std::mem::take(group);
*group = owned.description(description);
Ok(group.clone())
}
#[rhai_fn(name = "visibility", return_raw)]
pub fn set_visibility(
group: &mut RhaiGroup,
visibility: String,
) -> Result<RhaiGroup, Box<EvalAltResult>> {
let visibility_enum = match visibility.as_str() {
"Public" => Visibility::Public,
"Private" => Visibility::Private,
_ => return Err(format!("Invalid visibility: {}", visibility).into()),
};
let owned = std::mem::take(group);
*group = owned.visibility(visibility_enum);
Ok(group.clone())
}
#[rhai_fn(name = "save_group", return_raw)]
pub fn save_group(group: &mut RhaiGroup) -> Result<RhaiGroup, Box<EvalAltResult>> {
Ok(group.clone())
}
// Getters
#[rhai_fn(name = "get_id")]
pub fn get_id(group: &mut RhaiGroup) -> i64 {
group.base_data.id as i64
}
#[rhai_fn(name = "get_name")]
pub fn get_name(group: &mut RhaiGroup) -> String {
group.name.clone().unwrap_or_else(|| String::new())
}
#[rhai_fn(name = "get_description")]
pub fn get_description(group: &mut RhaiGroup) -> String {
group.description.clone().unwrap_or_else(|| String::new())
}
}
// ============================================================================
// Account Module (from money.rs)
// ============================================================================
type RhaiAccount = Account;
#[export_module]
mod rhai_account_module {
use super::RhaiAccount;
#[rhai_fn(name = "new_account", return_raw)]
pub fn new_account() -> Result<RhaiAccount, Box<EvalAltResult>> {
Ok(Account::new(0))
}
#[rhai_fn(name = "owner_id", return_raw)]
pub fn set_owner_id(
account: &mut RhaiAccount,
owner_id: i64,
) -> Result<RhaiAccount, Box<EvalAltResult>> {
let owned = std::mem::take(account);
*account = owned.owner_id(owner_id as u32);
Ok(account.clone())
}
#[rhai_fn(name = "address", return_raw)]
pub fn set_address(
account: &mut RhaiAccount,
address: String,
) -> Result<RhaiAccount, Box<EvalAltResult>> {
let owned = std::mem::take(account);
*account = owned.address(address);
Ok(account.clone())
}
#[rhai_fn(name = "currency", return_raw)]
pub fn set_currency(
account: &mut RhaiAccount,
currency: String,
) -> Result<RhaiAccount, Box<EvalAltResult>> {
let owned = std::mem::take(account);
*account = owned.currency(currency);
Ok(account.clone())
}
#[rhai_fn(name = "save_account", return_raw)]
pub fn save_account(account: &mut RhaiAccount) -> Result<RhaiAccount, Box<EvalAltResult>> {
Ok(account.clone())
}
// Getters
#[rhai_fn(name = "get_id")]
pub fn get_id(account: &mut RhaiAccount) -> i64 {
account.base_data.id as i64
}
#[rhai_fn(name = "get_address")]
pub fn get_address(account: &mut RhaiAccount) -> String {
account.address.clone()
}
#[rhai_fn(name = "get_currency")]
pub fn get_currency(account: &mut RhaiAccount) -> String {
account.currency.clone()
}
}
// ============================================================================
// DNS Zone Module
// ============================================================================
type RhaiDNSZone = DNSZone;
#[export_module]
mod rhai_dns_zone_module {
use super::RhaiDNSZone;
#[rhai_fn(name = "new_dns_zone", return_raw)]
pub fn new_dns_zone() -> Result<RhaiDNSZone, Box<EvalAltResult>> {
Ok(DNSZone::new(0))
}
#[rhai_fn(name = "name", return_raw)]
pub fn set_name(
zone: &mut RhaiDNSZone,
name: String,
) -> Result<RhaiDNSZone, Box<EvalAltResult>> {
let owned = std::mem::take(zone);
*zone = owned.name(name);
Ok(zone.clone())
}
#[rhai_fn(name = "description", return_raw)]
pub fn set_description(
zone: &mut RhaiDNSZone,
description: String,
) -> Result<RhaiDNSZone, Box<EvalAltResult>> {
let owned = std::mem::take(zone);
*zone = owned.description(description);
Ok(zone.clone())
}
#[rhai_fn(name = "save_dns_zone", return_raw)]
pub fn save_dns_zone(zone: &mut RhaiDNSZone) -> Result<RhaiDNSZone, Box<EvalAltResult>> {
Ok(zone.clone())
}
// Setters
#[rhai_fn(name = "set_domain")]
pub fn set_domain(zone: &mut RhaiDNSZone, domain: &str) {
let owned = std::mem::take(zone);
*zone = owned.domain(domain);
}
// Getters
#[rhai_fn(name = "get_id")]
pub fn get_id(zone: &mut RhaiDNSZone) -> i64 {
zone.base_data.id as i64
}
#[rhai_fn(name = "get_domain")]
pub fn get_domain(zone: &mut RhaiDNSZone) -> String {
zone.domain.clone()
}
}
// ============================================================================
// Registration Functions
// ============================================================================
// Registration functions
pub fn register_user_functions(engine: &mut Engine) {
let module = exported_module!(user_module);
engine.register_static_module("user", module.into());
}
pub fn register_group_functions(engine: &mut Engine) {
let module = exported_module!(group_module);
engine.register_static_module("group", module.into());
}
pub fn register_account_functions(engine: &mut Engine) {
let module = exported_module!(account_module);
engine.register_static_module("account", module.into());
}
pub fn register_dnszone_functions(engine: &mut Engine) {
let module = exported_module!(dnszone_module);
engine.register_static_module("dnszone", module.into());
}
/// Register all heroledger Rhai modules with the engine
pub fn register_heroledger_rhai_modules(engine: &mut Engine) {
register_user_functions(engine);
register_group_functions(engine);
register_account_functions(engine);
register_dnszone_functions(engine);
}

5
heromodels/src/models/object/mod.rs
View File

@@ -1,5 +1,6 @@
// Export contact module
// Export object module
pub mod object;
pub mod object_rhai_dsl;
// Re-export contact, Group from the inner contact module (contact.rs) within src/models/contact/mod.rs
// Re-export Object from the inner object module (object.rs) within src/models/object/mod.rs
pub use self::object::Object;

56
heromodels/src/models/object/object_rhai_dsl.rs Normal file
View File

@@ -0,0 +1,56 @@
use rhai::plugin::*;
use rhai::{CustomType, Dynamic, Engine, EvalAltResult, Module};
use super::Object;
type RhaiObject = Object;
#[export_module]
pub mod generated_rhai_module {
use super::*;
/// Create a new Object
#[rhai_fn(name = "new_object")]
pub fn new_object() -> RhaiObject {
Object::new()
}
/// Set the title of an Object
#[rhai_fn(name = "object_title")]
pub fn object_title(
object: &mut RhaiObject,
title: String,
) -> RhaiObject {
let mut result = object.clone();
result.title = title;
result
}
/// Set the description of an Object
#[rhai_fn(name = "object_description")]
pub fn object_description(
object: &mut RhaiObject,
description: String,
) -> RhaiObject {
let mut result = object.clone();
result.description = description;
result
}
/// Get the ID of an Object
#[rhai_fn(name = "get_object_id")]
pub fn get_object_id(object: &mut RhaiObject) -> i64 {
object.id() as i64
}
/// Get the title of an Object
#[rhai_fn(name = "get_object_title")]
pub fn get_object_title(object: &mut RhaiObject) -> String {
object.title.clone()
}
/// Get the description of an Object
#[rhai_fn(name = "get_object_description")]
pub fn get_object_description(object: &mut RhaiObject) -> String {
object.description.clone()
}
}

277
ourdb/API.md
View File

@@ -1,277 +0,0 @@
# OurDB API Reference
This document provides a comprehensive reference for the OurDB Rust API.
## Table of Contents
1. [Configuration](#configuration)
2. [Database Operations](#database-operations)
- [Creating and Opening](#creating-and-opening)
- [Setting Data](#setting-data)
- [Getting Data](#getting-data)
- [Deleting Data](#deleting-data)
- [History Tracking](#history-tracking)
3. [Error Handling](#error-handling)
4. [Advanced Usage](#advanced-usage)
- [Custom File Size](#custom-file-size)
- [Custom Key Size](#custom-key-size)
5. [Performance Considerations](#performance-considerations)
## Configuration
### OurDBConfig
The `OurDBConfig` struct is used to configure a new OurDB instance.
```rust
pub struct OurDBConfig {
pub path: PathBuf,
pub incremental_mode: bool,
pub file_size: Option<usize>,
pub keysize: Option<u8>,
}
```
| Field | Type | Description |
|-------|------|-------------|
| `path` | `PathBuf` | Path to the database directory |
| `incremental_mode` | `bool` | Whether to use auto-incremented IDs (true) or user-provided IDs (false) |
| `file_size` | `Option<usize>` | Maximum size of each database file in bytes (default: 500MB) |
| `keysize` | `Option<u8>` | Size of keys in bytes (default: 4, valid values: 2, 3, 4, 6) |
Example:
```rust
let config = OurDBConfig {
path: PathBuf::from("/path/to/db"),
incremental_mode: true,
file_size: Some(1024 * 1024 * 100), // 100MB
keysize: Some(4), // 4-byte keys
};
```
## Database Operations
### Creating and Opening
#### `OurDB::new`
Creates a new OurDB instance or opens an existing one.
```rust
pub fn new(config: OurDBConfig) -> Result<OurDB, Error>
```
Example:
```rust
let mut db = OurDB::new(config)?;
```
### Setting Data
#### `OurDB::set`
Sets a value in the database. In incremental mode, if no ID is provided, a new ID is generated.
```rust
pub fn set(&mut self, args: OurDBSetArgs) -> Result<u32, Error>
```
The `OurDBSetArgs` struct has the following fields:
```rust
pub struct OurDBSetArgs<'a> {
pub id: Option<u32>,
pub data: &'a [u8],
}
```
Example with auto-generated ID:
```rust
let id = db.set(OurDBSetArgs {
id: None,
data: b"Hello, World!",
})?;
```
Example with explicit ID:
```rust
db.set(OurDBSetArgs {
id: Some(42),
data: b"Hello, World!",
})?;
```
### Getting Data
#### `OurDB::get`
Retrieves a value from the database by ID.
```rust
pub fn get(&mut self, id: u32) -> Result<Vec<u8>, Error>
```
Example:
```rust
let data = db.get(42)?;
```
### Deleting Data
#### `OurDB::delete`
Deletes a value from the database by ID.
```rust
pub fn delete(&mut self, id: u32) -> Result<(), Error>
```
Example:
```rust
db.delete(42)?;
```
### History Tracking
#### `OurDB::get_history`
Retrieves the history of values for a given ID, up to the specified depth.
```rust
pub fn get_history(&mut self, id: u32, depth: u8) -> Result<Vec<Vec<u8>>, Error>
```
Example:
```rust
// Get the last 5 versions of the record
let history = db.get_history(42, 5)?;
// Process each version (most recent first)
for (i, version) in history.iter().enumerate() {
println!("Version {}: {:?}", i, version);
}
```
### Other Operations
#### `OurDB::get_next_id`
Returns the next ID that will be assigned in incremental mode.
```rust
pub fn get_next_id(&self) -> Result<u32, Error>
```
Example:
```rust
let next_id = db.get_next_id()?;
```
#### `OurDB::close`
Closes the database, ensuring all data is flushed to disk.
```rust
pub fn close(&mut self) -> Result<(), Error>
```
Example:
```rust
db.close()?;
```
#### `OurDB::destroy`
Closes the database and deletes all database files.
```rust
pub fn destroy(&mut self) -> Result<(), Error>
```
Example:
```rust
db.destroy()?;
```
## Error Handling
OurDB uses the `thiserror` crate to define error types. The main error type is `ourdb::Error`.
```rust
pub enum Error {
IoError(std::io::Error),
InvalidKeySize,
InvalidId,
RecordNotFound,
InvalidCrc,
NotIncrementalMode,
DatabaseClosed,
// ...
}
```
All OurDB operations that can fail return a `Result<T, Error>` which can be handled using Rust's standard error handling mechanisms.
Example:
```rust
match db.get(42) {
Ok(data) => println!("Found data: {:?}", data),
Err(ourdb::Error::RecordNotFound) => println!("Record not found"),
Err(e) => eprintln!("Error: {}", e),
}
```
## Advanced Usage
### Custom File Size
You can configure the maximum size of each database file:
```rust
let config = OurDBConfig {
path: PathBuf::from("/path/to/db"),
incremental_mode: true,
file_size: Some(1024 * 1024 * 10), // 10MB per file
keysize: None,
};
```
Smaller file sizes can be useful for:
- Limiting memory usage when reading files
- Improving performance on systems with limited memory
- Easier backup and file management
### Custom Key Size
OurDB supports different key sizes (2, 3, 4, or 6 bytes):
```rust
let config = OurDBConfig {
path: PathBuf::from("/path/to/db"),
incremental_mode: true,
file_size: None,
keysize: Some(6), // 6-byte keys
};
```
Key size considerations:
- 2 bytes: Up to 65,536 records
- 3 bytes: Up to 16,777,216 records
- 4 bytes: Up to 4,294,967,296 records (default)
- 6 bytes: Up to 281,474,976,710,656 records
## Performance Considerations
For optimal performance:
1. **Choose appropriate key size**: Use the smallest key size that can accommodate your expected number of records.
2. **Configure file size**: For large databases, consider using smaller file sizes to improve memory usage.
3. **Batch operations**: When inserting or updating many records, consider batching operations to minimize disk I/O.
4. **Close properly**: Always call `close()` when you're done with the database to ensure data is properly flushed to disk.
5. **Reuse OurDB instance**: Creating a new OurDB instance has overhead, so reuse the same instance for multiple operations when possible.
6. **Consider memory usage**: The lookup table is loaded into memory, so very large databases may require significant RAM.

806
ourdb/Cargo.lock generated
View File

@@ -1,806 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "aho-corasick"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
"memchr",
]
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstyle"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
[[package]]
name = "autocfg"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "bitflags"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
[[package]]
name = "bumpalo"
version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "ciborium"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
[[package]]
name = "ciborium-ll"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "clap"
version = "4.5.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.5.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9"
dependencies = [
"anstyle",
"clap_lex",
]
[[package]]
name = "clap_lex"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "crc32fast"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
dependencies = [
"anes",
"cast",
"ciborium",
"clap",
"criterion-plot",
"is-terminal",
"itertools",
"num-traits",
"once_cell",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "errno"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "getrandom"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasi 0.14.2+wasi-0.2.4",
]
[[package]]
name = "half"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9"
dependencies = [
"cfg-if",
"crunchy",
]
[[package]]
name = "hermit-abi"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e"
[[package]]
name = "is-terminal"
version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi",
"libc",
"windows-sys",
]
[[package]]
name = "itertools"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "js-sys"
version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
dependencies = [
"once_cell",
"wasm-bindgen",
]
[[package]]
name = "libc"
version = "0.2.171"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
[[package]]
name = "linux-raw-sys"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12"
[[package]]
name = "log"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "memchr"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "oorandom"
version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "ourdb"
version = "0.1.0"
dependencies = [
"crc32fast",
"criterion",
"log",
"rand",
"tempfile",
"thiserror",
]
[[package]]
name = "plotters"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
[[package]]
name = "plotters-svg"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
dependencies = [
"plotters-backend",
]
[[package]]
name = "ppv-lite86"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy",
]
[[package]]
name = "proc-macro2"
version = "1.0.94"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom 0.2.15",
]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "regex"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rustix"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "rustversion"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
[[package]]
name = "ryu"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "serde"
version = "1.0.219"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.219"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.140"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
[[package]]
name = "syn"
version = "2.0.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf"
dependencies = [
"fastrand",
"getrandom 0.3.2",
"once_cell",
"rustix",
"windows-sys",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "unicode-ident"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasi"
version = "0.14.2+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
dependencies = [
"wit-bindgen-rt",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
dependencies = [
"bumpalo",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
dependencies = [
"unicode-ident",
]
[[package]]
name = "web-sys"
version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "winapi-util"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "wit-bindgen-rt"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
"bitflags",
]
[[package]]
name = "zerocopy"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
dependencies = [
"proc-macro2",
"quote",
"syn",
]

32
ourdb/Cargo.toml
View File

@@ -1,32 +0,0 @@
[package]
name = "ourdb"
version = "0.1.0"
edition = "2021"
description = "A lightweight, efficient key-value database with history tracking capabilities"
authors = ["OurWorld Team"]
[dependencies]
crc32fast = "1.3.2"
thiserror = "1.0.40"
log = "0.4.17"
rand = "0.8.5"
[dev-dependencies]
criterion = "0.5.1"
tempfile = "3.8.0"
# [[bench]]
# name = "ourdb_benchmarks"
# harness = false
[[example]]
name = "basic_usage"
path = "examples/basic_usage.rs"
[[example]]
name = "advanced_usage"
path = "examples/advanced_usage.rs"
[[example]]
name = "benchmark"
path = "examples/benchmark.rs"

135
ourdb/README.md
View File

@@ -1,135 +0,0 @@
# OurDB
OurDB is a lightweight, efficient key-value database implementation that provides data persistence with history tracking capabilities. This Rust implementation offers a robust and performant solution for applications requiring simple but reliable data storage.
## Features
- Simple key-value storage with history tracking
- Data integrity verification using CRC32
- Support for multiple backend files for large datasets
- Lookup table for fast data retrieval
- Incremental mode for auto-generated IDs
- Memory and disk-based lookup tables
## Limitations
- Maximum data size per entry is 65,535 bytes (~64KB) due to the 2-byte size field in the record header
## Usage
### Basic Example
```rust
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use std::path::PathBuf;
fn main() -> Result<(), ourdb::Error> {
// Create a new database
let config = OurDBConfig {
path: PathBuf::from("/tmp/ourdb"),
incremental_mode: true,
file_size: None, // Use default (500MB)
keysize: None, // Use default (4 bytes)
};
let mut db = OurDB::new(config)?;
// Store data (with auto-generated ID in incremental mode)
let data = b"Hello, OurDB!";
let id = db.set(OurDBSetArgs { id: None, data })?;
println!("Stored data with ID: {}", id);
// Retrieve data
let retrieved = db.get(id)?;
println!("Retrieved: {}", String::from_utf8_lossy(&retrieved));
// Update data
let updated_data = b"Updated data";
db.set(OurDBSetArgs { id: Some(id), data: updated_data })?;
// Get history (returns most recent first)
let history = db.get_history(id, 2)?;
for (i, entry) in history.iter().enumerate() {
println!("History {}: {}", i, String::from_utf8_lossy(entry));
}
// Delete data
db.delete(id)?;
// Close the database
db.close()?;
Ok(())
}
```
### Key-Value Mode vs Incremental Mode
OurDB supports two operating modes:
1. **Key-Value Mode** (`incremental_mode: false`): You must provide IDs explicitly when storing data.
2. **Incremental Mode** (`incremental_mode: true`): IDs are auto-generated when not provided.
### Configuration Options
- `path`: Directory for database storage
- `incremental_mode`: Whether to use auto-increment mode
- `file_size`: Maximum file size (default: 500MB)
- `keysize`: Size of lookup table entries (2-6 bytes)
- 2: For databases with < 65,536 records
- 3: For databases with < 16,777,216 records
- 4: For databases with < 4,294,967,296 records (default)
- 6: For large databases requiring multiple files
## Architecture
OurDB consists of three main components:
1. **Frontend API**: Provides the public interface for database operations
2. **Lookup Table**: Maps keys to physical locations in the backend storage
3. **Backend Storage**: Manages the actual data persistence in files
### Record Format
Each record in the backend storage includes:
- 2 bytes: Data size
- 4 bytes: CRC32 checksum
- 6 bytes: Previous record location (for history)
- N bytes: Actual data
## Documentation
Additional documentation is available in the repository:
- [API Reference](API.md): Detailed API documentation
- [Migration Guide](MIGRATION.md): Guide for migrating from the V implementation
- [Architecture](architecture.md): Design and implementation details
## Examples
The repository includes several examples to demonstrate OurDB usage:
- `basic_usage.rs`: Simple operations with OurDB
- `advanced_usage.rs`: More complex features including both operation modes
- `benchmark.rs`: Performance benchmarking tool
Run an example with:
```bash
cargo run --example basic_usage
cargo run --example advanced_usage
cargo run --example benchmark
```
## Performance
OurDB is designed for efficiency and minimal overhead. The benchmark example can be used to evaluate performance on your specific hardware and workload.
Typical performance metrics on modern hardware:
- **Write**: 10,000+ operations per second
- **Read**: 50,000+ operations per second
## License
This project is licensed under the MIT License.

439
ourdb/architecture.md
View File

@@ -1,439 +0,0 @@
# OurDB: Architecture for V to Rust Port
## 1. Overview
OurDB is a lightweight, efficient key-value database implementation that provides data persistence with history tracking capabilities. This document outlines the architecture for porting OurDB from its original V implementation to Rust, maintaining all existing functionality while leveraging Rust's memory safety, performance, and ecosystem.
## 2. Current Architecture (V Implementation)
The current V implementation of OurDB consists of three main components in a layered architecture:
```mermaid
graph TD
A[Client Code] --> B[Frontend API]
B --> C[Lookup Table]
B --> D[Backend Storage]
C --> D
```
### 2.1 Frontend (db.v)
The frontend provides the public API for database operations and coordinates between the lookup table and backend storage components.
Key responsibilities:
- Exposing high-level operations (set, get, delete, history)
- Managing incremental ID generation in auto-increment mode
- Coordinating data flow between lookup and backend components
- Handling database lifecycle (open, close, destroy)
### 2.2 Lookup Table (lookup.v)
The lookup table maps keys to physical locations in the backend storage.
Key responsibilities:
- Maintaining key-to-location mapping
- Optimizing key sizes based on database configuration
- Supporting both memory and disk-based lookup tables
- Handling sparse data efficiently
- Providing next ID generation for incremental mode
### 2.3 Backend Storage (backend.v)
The backend storage manages the actual data persistence in files.
Key responsibilities:
- Managing physical data storage in files
- Ensuring data integrity with CRC32 checksums
- Supporting multiple file backends for large datasets
- Implementing low-level read/write operations
- Tracking record history through linked locations
### 2.4 Core Data Structures
#### OurDB
```v
@[heap]
pub struct OurDB {
mut:
lookup &LookupTable
pub:
path string // directory for storage
incremental_mode bool
file_size u32 = 500 * (1 << 20) // 500MB
pub mut:
file os.File
file_nr u16 // the file which is open
last_used_file_nr u16
}
```
#### LookupTable
```v
pub struct LookupTable {
keysize u8
lookuppath string
mut:
data []u8
incremental ?u32 // points to next empty slot if incremental mode is enabled
}
```
#### Location
```v
pub struct Location {
pub mut:
file_nr u16
position u32
}
```
### 2.5 Storage Format
#### Record Format
Each record in the backend storage includes:
- 2 bytes: Data size
- 4 bytes: CRC32 checksum
- 6 bytes: Previous record location (for history)
- N bytes: Actual data
#### Lookup Table Optimization
The lookup table automatically optimizes its key size based on the database configuration:
- 2 bytes: For databases with < 65,536 records
- 3 bytes: For databases with < 16,777,216 records
- 4 bytes: For databases with < 4,294,967,296 records
- 6 bytes: For large databases requiring multiple files
## 3. Proposed Rust Architecture
The Rust implementation will maintain the same layered architecture while leveraging Rust's type system, ownership model, and error handling.
```mermaid
graph TD
A[Client Code] --> B[OurDB API]
B --> C[LookupTable]
B --> D[Backend]
C --> D
E[Error Handling] --> B
E --> C
E --> D
F[Configuration] --> B
```
### 3.1 Core Components
#### 3.1.1 OurDB (API Layer)
```rust
pub struct OurDB {
path: String,
incremental_mode: bool,
file_size: u32,
lookup: LookupTable,
file: Option<std::fs::File>,
file_nr: u16,
last_used_file_nr: u16,
}
impl OurDB {
pub fn new(config: OurDBConfig) -> Result<Self, Error>;
pub fn set(&mut self, id: Option<u32>, data: &[u8]) -> Result<u32, Error>;
pub fn get(&mut self, id: u32) -> Result<Vec<u8>, Error>;
pub fn get_history(&mut self, id: u32, depth: u8) -> Result<Vec<Vec<u8>>, Error>;
pub fn delete(&mut self, id: u32) -> Result<(), Error>;
pub fn get_next_id(&mut self) -> Result<u32, Error>;
pub fn close(&mut self) -> Result<(), Error>;
pub fn destroy(&mut self) -> Result<(), Error>;
}
```
#### 3.1.2 LookupTable
```rust
pub struct LookupTable {
keysize: u8,
lookuppath: String,
data: Vec<u8>,
incremental: Option<u32>,
}
impl LookupTable {
fn new(config: LookupConfig) -> Result<Self, Error>;
fn get(&self, id: u32) -> Result<Location, Error>;
fn set(&mut self, id: u32, location: Location) -> Result<(), Error>;
fn delete(&mut self, id: u32) -> Result<(), Error>;
fn get_next_id(&self) -> Result<u32, Error>;
fn increment_index(&mut self) -> Result<(), Error>;
fn export_data(&self, path: &str) -> Result<(), Error>;
fn import_data(&mut self, path: &str) -> Result<(), Error>;
fn export_sparse(&self, path: &str) -> Result<(), Error>;
fn import_sparse(&mut self, path: &str) -> Result<(), Error>;
}
```
#### 3.1.3 Location
```rust
pub struct Location {
file_nr: u16,
position: u32,
}
impl Location {
fn new(bytes: &[u8], keysize: u8) -> Result<Self, Error>;
fn to_bytes(&self) -> Result<Vec<u8>, Error>;
fn to_u64(&self) -> u64;
}
```
#### 3.1.4 Backend
The backend functionality will be implemented as methods on the OurDB struct:
```rust
impl OurDB {
fn db_file_select(&mut self, file_nr: u16) -> Result<(), Error>;
fn create_new_db_file(&mut self, file_nr: u16) -> Result<(), Error>;
fn get_file_nr(&mut self) -> Result<u16, Error>;
fn set_(&mut self, id: u32, old_location: Location, data: &[u8]) -> Result<(), Error>;
fn get_(&mut self, location: Location) -> Result<Vec<u8>, Error>;
fn get_prev_pos_(&mut self, location: Location) -> Result<Location, Error>;
fn delete_(&mut self, id: u32, location: Location) -> Result<(), Error>;
fn close_(&mut self);
}
```
#### 3.1.5 Configuration
```rust
pub struct OurDBConfig {
pub record_nr_max: u32,
pub record_size_max: u32,
pub file_size: u32,
pub path: String,
pub incremental_mode: bool,
pub reset: bool,
}
struct LookupConfig {
size: u32,
keysize: u8,
lookuppath: String,
incremental_mode: bool,
}
```
#### 3.1.6 Error Handling
```rust
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
#[error("Invalid key size: {0}")]
InvalidKeySize(u8),
#[error("Record not found: {0}")]
RecordNotFound(u32),
#[error("Data corruption: CRC mismatch")]
DataCorruption,
#[error("Index out of bounds: {0}")]
IndexOutOfBounds(u32),
#[error("Incremental mode not enabled")]
IncrementalNotEnabled,
#[error("Lookup table is full")]
LookupTableFull,
#[error("Invalid file number: {0}")]
InvalidFileNumber(u16),
#[error("Invalid operation: {0}")]
InvalidOperation(String),
}
```
## 4. Implementation Strategy
### 4.1 Phase 1: Core Data Structures
1. Implement the `Location` struct with serialization/deserialization
2. Implement the `Error` enum for error handling
3. Implement the configuration structures
### 4.2 Phase 2: Lookup Table
1. Implement the `LookupTable` struct with memory-based storage
2. Add disk-based storage support
3. Implement key size optimization
4. Add incremental ID support
5. Implement import/export functionality
### 4.3 Phase 3: Backend Storage
1. Implement file management functions
2. Implement record serialization/deserialization with CRC32
3. Implement history tracking through linked locations
4. Add support for multiple backend files
### 4.4 Phase 4: Frontend API
1. Implement the `OurDB` struct with core operations
2. Add high-level API methods (set, get, delete, history)
3. Implement database lifecycle management
### 4.5 Phase 5: Testing and Optimization
1. Port existing tests from V to Rust
2. Add new tests for Rust-specific functionality
3. Benchmark and optimize performance
4. Ensure compatibility with existing OurDB files
## 5. Implementation Considerations
### 5.1 Memory Management
Leverage Rust's ownership model for safe and efficient memory management:
- Use `Vec<u8>` for data buffers instead of raw pointers
- Implement proper RAII for file handles
- Use references and borrows to avoid unnecessary copying
- Consider using `Bytes` from the `bytes` crate for zero-copy operations
### 5.2 Error Handling
Use Rust's `Result` type for comprehensive error handling:
- Define custom error types for OurDB-specific errors
- Propagate errors using the `?` operator
- Provide detailed error messages
- Implement proper error conversion using the `From` trait
### 5.3 File I/O
Optimize file operations for performance:
- Use `BufReader` and `BufWriter` for buffered I/O
- Implement proper file locking for concurrent access
- Consider memory-mapped files for lookup tables
- Use `seek` and `read_exact` for precise positioning
### 5.4 Concurrency
Consider thread safety for concurrent database access:
- Use interior mutability patterns where appropriate
- Implement `Send` and `Sync` traits for thread safety
- Consider using `RwLock` for shared read access
- Provide clear documentation on thread safety guarantees
### 5.5 Performance Optimizations
Identify opportunities for performance improvements:
- Use memory-mapped files for lookup tables
- Implement caching for frequently accessed records
- Use zero-copy operations where possible
- Consider async I/O for non-blocking operations
## 6. Testing Strategy
### 6.1 Unit Tests
Write comprehensive unit tests for each component:
- Test `Location` serialization/deserialization
- Test `LookupTable` operations
- Test backend storage functions
- Test error handling
### 6.2 Integration Tests
Write integration tests for the complete system:
- Test database creation and configuration
- Test basic CRUD operations
- Test history tracking
- Test incremental ID generation
- Test file management
### 6.3 Compatibility Tests
Ensure compatibility with existing OurDB files:
- Test reading existing V-created OurDB files
- Test writing files that can be read by the V implementation
- Test migration scenarios
### 6.4 Performance Tests
Benchmark performance against the V implementation:
- Measure throughput for set/get operations
- Measure latency for different operations
- Test with different database sizes
- Test with different record sizes
## 7. Project Structure
```
ourdb/
├── Cargo.toml
├── src/
│ ├── lib.rs # Public API and re-exports
│ ├── ourdb.rs # OurDB implementation (frontend)
│ ├── lookup.rs # Lookup table implementation
│ ├── location.rs # Location struct implementation
│ ├── backend.rs # Backend storage implementation
│ ├── error.rs # Error types
│ ├── config.rs # Configuration structures
│ └── utils.rs # Utility functions
├── tests/
│ ├── unit/ # Unit tests
│ ├── integration/ # Integration tests
│ └── compatibility/ # Compatibility tests
└── examples/
├── basic.rs # Basic usage example
├── history.rs # History tracking example
└── client_server.rs # Client-server example
```
## 8. Dependencies
The Rust implementation will use the following dependencies:
- `thiserror` for error handling
- `crc32fast` for CRC32 calculation
- `bytes` for efficient byte manipulation
- `memmap2` for memory-mapped files (optional)
- `serde` for serialization (optional, for future extensions)
- `log` for logging
- `criterion` for benchmarking
## 9. Compatibility Considerations
To ensure compatibility with the V implementation:
1. Maintain the same file format for data storage
2. Preserve the lookup table format
3. Keep the same CRC32 calculation method
4. Ensure identical behavior for incremental ID generation
5. Maintain the same history tracking mechanism
## 10. Future Extensions
Potential future extensions to consider:
1. Async API for non-blocking operations
2. Transactions support
3. Better concurrency control
4. Compression support
5. Encryption support
6. Streaming API for large values
7. Iterators for scanning records
8. Secondary indexes
## 11. Conclusion
This architecture provides a roadmap for porting OurDB from V to Rust while maintaining compatibility and leveraging Rust's strengths. The implementation will follow a phased approach, starting with core data structures and gradually building up to the complete system.
The Rust implementation aims to be:
- **Safe**: Leveraging Rust's ownership model for memory safety
- **Fast**: Maintaining or improving performance compared to V
- **Compatible**: Working with existing OurDB files
- **Extensible**: Providing a foundation for future enhancements
- **Well-tested**: Including comprehensive test coverage

231
ourdb/examples/advanced_usage.rs
View File

@@ -1,231 +0,0 @@
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use std::path::PathBuf;
use std::time::Instant;
fn main() -> Result<(), ourdb::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("ourdb_advanced_example");
std::fs::create_dir_all(&db_path)?;
println!("Creating database at: {}", db_path.display());
// Demonstrate key-value mode (non-incremental)
key_value_mode_example(&db_path)?;
// Demonstrate incremental mode
incremental_mode_example(&db_path)?;
// Demonstrate performance benchmarking
performance_benchmark(&db_path)?;
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("Cleaned up database directory");
} else {
println!("Database kept at: {}", db_path.display());
}
Ok(())
}
fn key_value_mode_example(base_path: &PathBuf) -> Result<(), ourdb::Error> {
println!("\n=== Key-Value Mode Example ===");
let db_path = base_path.join("key_value");
std::fs::create_dir_all(&db_path)?;
// Create a new database with key-value mode (non-incremental)
let config = OurDBConfig {
path: db_path,
incremental_mode: false,
file_size: Some(1024 * 1024), // 1MB for testing
keysize: Some(2), // Small key size for demonstration
reset: None, // Don't reset existing database
};
let mut db = OurDB::new(config)?;
// In key-value mode, we must provide IDs explicitly
let custom_ids = [100, 200, 300, 400, 500];
// Store data with custom IDs
for (i, &id) in custom_ids.iter().enumerate() {
let data = format!("Record with custom ID {}", id);
db.set(OurDBSetArgs {
id: Some(id),
data: data.as_bytes(),
})?;
println!("Stored record {} with custom ID: {}", i + 1, id);
}
// Retrieve data by custom IDs
for &id in &custom_ids {
let retrieved = db.get(id)?;
println!(
"Retrieved ID {}: {}",
id,
String::from_utf8_lossy(&retrieved)
);
}
// Update and track history
let id_to_update = custom_ids[2]; // ID 300
for i in 1..=3 {
let updated_data = format!("Updated record {} (version {})", id_to_update, i);
db.set(OurDBSetArgs {
id: Some(id_to_update),
data: updated_data.as_bytes(),
})?;
println!("Updated ID {} (version {})", id_to_update, i);
}
// Get history for the updated record
let history = db.get_history(id_to_update, 5)?;
println!("History for ID {} (most recent first):", id_to_update);
for (i, entry) in history.iter().enumerate() {
println!(" Version {}: {}", i, String::from_utf8_lossy(entry));
}
db.close()?;
println!("Key-value mode example completed");
Ok(())
}
fn incremental_mode_example(base_path: &PathBuf) -> Result<(), ourdb::Error> {
println!("\n=== Incremental Mode Example ===");
let db_path = base_path.join("incremental");
std::fs::create_dir_all(&db_path)?;
// Create a new database with incremental mode
let config = OurDBConfig {
path: db_path,
incremental_mode: true,
file_size: Some(1024 * 1024), // 1MB for testing
keysize: Some(3), // 3-byte keys
reset: None, // Don't reset existing database
};
let mut db = OurDB::new(config)?;
// In incremental mode, IDs are auto-generated
let mut assigned_ids = Vec::new();
// Store multiple records and collect assigned IDs
for i in 1..=5 {
let data = format!("Auto-increment record {}", i);
let id = db.set(OurDBSetArgs {
id: None,
data: data.as_bytes(),
})?;
assigned_ids.push(id);
println!("Stored record {} with auto-assigned ID: {}", i, id);
}
// Check next ID
let next_id = db.get_next_id()?;
println!("Next ID to be assigned: {}", next_id);
// Retrieve all records
for &id in &assigned_ids {
let retrieved = db.get(id)?;
println!(
"Retrieved ID {}: {}",
id,
String::from_utf8_lossy(&retrieved)
);
}
db.close()?;
println!("Incremental mode example completed");
Ok(())
}
fn performance_benchmark(base_path: &PathBuf) -> Result<(), ourdb::Error> {
println!("\n=== Performance Benchmark ===");
let db_path = base_path.join("benchmark");
std::fs::create_dir_all(&db_path)?;
// Create a new database
let config = OurDBConfig {
path: db_path,
incremental_mode: true,
file_size: Some(1024 * 1024), // 10MB
keysize: Some(4), // 4-byte keys
reset: None, // Don't reset existing database
};
let mut db = OurDB::new(config)?;
// Number of operations for the benchmark
let num_operations = 1000;
let data_size = 100; // bytes per record
// Prepare test data
let test_data = vec![b'A'; data_size];
// Benchmark write operations
println!("Benchmarking {} write operations...", num_operations);
let start = Instant::now();
let mut ids = Vec::with_capacity(num_operations);
for _ in 0..num_operations {
let id = db.set(OurDBSetArgs {
id: None,
data: &test_data,
})?;
ids.push(id);
}
let write_duration = start.elapsed();
let writes_per_second = num_operations as f64 / write_duration.as_secs_f64();
println!(
"Write performance: {:.2} ops/sec ({:.2} ms/op)",
writes_per_second,
write_duration.as_secs_f64() * 1000.0 / num_operations as f64
);
// Benchmark read operations
println!("Benchmarking {} read operations...", num_operations);
let start = Instant::now();
for &id in &ids {
let _ = db.get(id)?;
}
let read_duration = start.elapsed();
let reads_per_second = num_operations as f64 / read_duration.as_secs_f64();
println!(
"Read performance: {:.2} ops/sec ({:.2} ms/op)",
reads_per_second,
read_duration.as_secs_f64() * 1000.0 / num_operations as f64
);
// Benchmark update operations
println!("Benchmarking {} update operations...", num_operations);
let start = Instant::now();
for &id in &ids {
db.set(OurDBSetArgs {
id: Some(id),
data: &test_data,
})?;
}
let update_duration = start.elapsed();
let updates_per_second = num_operations as f64 / update_duration.as_secs_f64();
println!(
"Update performance: {:.2} ops/sec ({:.2} ms/op)",
updates_per_second,
update_duration.as_secs_f64() * 1000.0 / num_operations as f64
);
db.close()?;
println!("Performance benchmark completed");
Ok(())
}

89
ourdb/examples/basic_usage.rs
View File

@@ -1,89 +0,0 @@
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
fn main() -> Result<(), ourdb::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("ourdb_example");
std::fs::create_dir_all(&db_path)?;
println!("Creating database at: {}", db_path.display());
// Create a new database with incremental mode enabled
let config = OurDBConfig {
path: db_path.clone(),
incremental_mode: true,
file_size: None, // Use default (500MB)
keysize: None, // Use default (4 bytes)
reset: None, // Don't reset existing database
};
let mut db = OurDB::new(config)?;
// Store some data with auto-generated IDs
let data1 = b"First record";
let id1 = db.set(OurDBSetArgs {
id: None,
data: data1,
})?;
println!("Stored first record with ID: {}", id1);
let data2 = b"Second record";
let id2 = db.set(OurDBSetArgs {
id: None,
data: data2,
})?;
println!("Stored second record with ID: {}", id2);
// Retrieve and print the data
let retrieved1 = db.get(id1)?;
println!(
"Retrieved ID {}: {}",
id1,
String::from_utf8_lossy(&retrieved1)
);
let retrieved2 = db.get(id2)?;
println!(
"Retrieved ID {}: {}",
id2,
String::from_utf8_lossy(&retrieved2)
);
// Update a record to demonstrate history tracking
let updated_data = b"Updated first record";
db.set(OurDBSetArgs {
id: Some(id1),
data: updated_data,
})?;
println!("Updated record with ID: {}", id1);
// Get history for the updated record
let history = db.get_history(id1, 2)?;
println!("History for ID {}:", id1);
for (i, entry) in history.iter().enumerate() {
println!(" Version {}: {}", i, String::from_utf8_lossy(entry));
}
// Delete a record
db.delete(id2)?;
println!("Deleted record with ID: {}", id2);
// Verify deletion
match db.get(id2) {
Ok(_) => println!("Record still exists (unexpected)"),
Err(e) => println!("Verified deletion: {}", e),
}
// Close the database
db.close()?;
println!("Database closed successfully");
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("Cleaned up database directory");
} else {
println!("Database kept at: {}", db_path.display());
}
Ok(())
}

124
ourdb/examples/benchmark.rs
View File

@@ -1,124 +0,0 @@
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use std::time::Instant;
fn main() -> Result<(), ourdb::Error> {
// Parse command-line arguments
let args: Vec<String> = std::env::args().collect();
// Default values
let mut incremental_mode = true;
let mut keysize: u8 = 4;
let mut num_operations = 10000;
// Parse arguments
for i in 1..args.len() {
if args[i] == "--no-incremental" {
incremental_mode = false;
} else if args[i] == "--keysize" && i + 1 < args.len() {
keysize = args[i + 1].parse().unwrap_or(4);
} else if args[i] == "--ops" && i + 1 < args.len() {
num_operations = args[i + 1].parse().unwrap_or(10000);
}
}
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("ourdb_benchmark");
std::fs::create_dir_all(&db_path)?;
println!("Database path: {}", db_path.display());
// Create a new database
let config = OurDBConfig {
path: db_path.clone(),
incremental_mode,
file_size: Some(1024 * 1024),
keysize: Some(keysize),
reset: Some(true), // Reset the database for benchmarking
};
let mut db = OurDB::new(config)?;
// Prepare test data (100 bytes per record)
let test_data = vec![b'A'; 100];
// Benchmark write operations
println!(
"Benchmarking {} write operations (incremental: {}, keysize: {})...",
num_operations, incremental_mode, keysize
);
let start = Instant::now();
let mut ids = Vec::with_capacity(num_operations);
for _ in 0..num_operations {
let id = if incremental_mode {
db.set(OurDBSetArgs {
id: None,
data: &test_data,
})?
} else {
// In non-incremental mode, we need to provide IDs
let id = ids.len() as u32 + 1;
db.set(OurDBSetArgs {
id: Some(id),
data: &test_data,
})?;
id
};
ids.push(id);
}
let write_duration = start.elapsed();
let writes_per_second = num_operations as f64 / write_duration.as_secs_f64();
println!(
"Write performance: {:.2} ops/sec ({:.2} ms/op)",
writes_per_second,
write_duration.as_secs_f64() * 1000.0 / num_operations as f64
);
// Benchmark read operations
println!("Benchmarking {} read operations...", num_operations);
let start = Instant::now();
for &id in &ids {
let _ = db.get(id)?;
}
let read_duration = start.elapsed();
let reads_per_second = num_operations as f64 / read_duration.as_secs_f64();
println!(
"Read performance: {:.2} ops/sec ({:.2} ms/op)",
reads_per_second,
read_duration.as_secs_f64() * 1000.0 / num_operations as f64
);
// Benchmark update operations
println!("Benchmarking {} update operations...", num_operations);
let start = Instant::now();
for &id in &ids {
db.set(OurDBSetArgs {
id: Some(id),
data: &test_data,
})?;
}
let update_duration = start.elapsed();
let updates_per_second = num_operations as f64 / update_duration.as_secs_f64();
println!(
"Update performance: {:.2} ops/sec ({:.2} ms/op)",
updates_per_second,
update_duration.as_secs_f64() * 1000.0 / num_operations as f64
);
// Clean up
db.close()?;
std::fs::remove_dir_all(&db_path)?;
Ok(())
}

83
ourdb/examples/main.rs
View File

@@ -1,83 +0,0 @@
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use std::env::temp_dir;
use std::time::{SystemTime, UNIX_EPOCH};
fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("Standalone OurDB Example");
println!("=======================\n");
// Create a temporary directory for the database
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
let db_path = temp_dir().join(format!("ourdb_example_{}", timestamp));
std::fs::create_dir_all(&db_path)?;
println!("Creating database at: {}", db_path.display());
// Create a new OurDB instance
let config = OurDBConfig {
path: db_path.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: Some(false),
};
let mut db = OurDB::new(config)?;
println!("Database created successfully");
// Store some data
let test_data = b"Hello, OurDB!";
let id = db.set(OurDBSetArgs {
id: None,
data: test_data,
})?;
println!("\nStored data with ID: {}", id);
// Retrieve the data
let retrieved = db.get(id)?;
println!("Retrieved data: {}", String::from_utf8_lossy(&retrieved));
// Update the data
let updated_data = b"Updated data in OurDB!";
db.set(OurDBSetArgs {
id: Some(id),
data: updated_data,
})?;
println!("\nUpdated data with ID: {}", id);
// Retrieve the updated data
let retrieved = db.get(id)?;
println!(
"Retrieved updated data: {}",
String::from_utf8_lossy(&retrieved)
);
// Get history
let history = db.get_history(id, 2)?;
println!("\nHistory for ID {}:", id);
for (i, data) in history.iter().enumerate() {
println!(" Version {}: {}", i + 1, String::from_utf8_lossy(data));
}
// Delete the data
db.delete(id)?;
println!("\nDeleted data with ID: {}", id);
// Try to retrieve the deleted data (should fail)
match db.get(id) {
Ok(_) => println!("Data still exists (unexpected)"),
Err(e) => println!("Verified deletion: {}", e),
}
println!("\nExample completed successfully!");
// Clean up
db.close()?;
std::fs::remove_dir_all(&db_path)?;
println!("Cleaned up database directory");
Ok(())
}

83
ourdb/examples/standalone_ourdb_example.rs
View File

@@ -1,83 +0,0 @@
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use std::env::temp_dir;
use std::time::{SystemTime, UNIX_EPOCH};
fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("Standalone OurDB Example");
println!("=======================\n");
// Create a temporary directory for the database
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
let db_path = temp_dir().join(format!("ourdb_example_{}", timestamp));
std::fs::create_dir_all(&db_path)?;
println!("Creating database at: {}", db_path.display());
// Create a new OurDB instance
let config = OurDBConfig {
path: db_path.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: Some(false),
};
let mut db = OurDB::new(config)?;
println!("Database created successfully");
// Store some data
let test_data = b"Hello, OurDB!";
let id = db.set(OurDBSetArgs {
id: None,
data: test_data,
})?;
println!("\nStored data with ID: {}", id);
// Retrieve the data
let retrieved = db.get(id)?;
println!("Retrieved data: {}", String::from_utf8_lossy(&retrieved));
// Update the data
let updated_data = b"Updated data in OurDB!";
db.set(OurDBSetArgs {
id: Some(id),
data: updated_data,
})?;
println!("\nUpdated data with ID: {}", id);
// Retrieve the updated data
let retrieved = db.get(id)?;
println!(
"Retrieved updated data: {}",
String::from_utf8_lossy(&retrieved)
);
// Get history
let history = db.get_history(id, 2)?;
println!("\nHistory for ID {}:", id);
for (i, data) in history.iter().enumerate() {
println!(" Version {}: {}", i + 1, String::from_utf8_lossy(data));
}
// Delete the data
db.delete(id)?;
println!("\nDeleted data with ID: {}", id);
// Try to retrieve the deleted data (should fail)
match db.get(id) {
Ok(_) => println!("Data still exists (unexpected)"),
Err(e) => println!("Verified deletion: {}", e),
}
println!("\nExample completed successfully!");
// Clean up
db.close()?;
std::fs::remove_dir_all(&db_path)?;
println!("Cleaned up database directory");
Ok(())
}

366
ourdb/src/backend.rs
View File

@@ -1,366 +0,0 @@
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use crc32fast::Hasher;
use crate::error::Error;
use crate::location::Location;
use crate::OurDB;
// Header size: 2 bytes (size) + 4 bytes (CRC32) + 6 bytes (previous location)
pub const HEADER_SIZE: usize = 12;
impl OurDB {
/// Selects and opens a database file for read/write operations
pub(crate) fn db_file_select(&mut self, file_nr: u16) -> Result<(), Error> {
// No need to check if file_nr > 65535 as u16 can't exceed that value
let path = self.path.join(format!("{}.db", file_nr));
// Always close the current file if it's open
self.file = None;
// Create file if it doesn't exist
if !path.exists() {
self.create_new_db_file(file_nr)?;
}
// Open the file fresh
let file = OpenOptions::new().read(true).write(true).open(&path)?;
self.file = Some(file);
self.file_nr = file_nr;
Ok(())
}
/// Creates a new database file
pub(crate) fn create_new_db_file(&mut self, file_nr: u16) -> Result<(), Error> {
let new_file_path = self.path.join(format!("{}.db", file_nr));
let mut file = File::create(&new_file_path)?;
// Write a single byte to make all positions start from 1
file.write_all(&[0u8])?;
Ok(())
}
/// Gets the file number to use for the next write operation
pub(crate) fn get_file_nr(&mut self) -> Result<u16, Error> {
// For keysize 2, 3, or 4, we can only use file_nr 0
if self.lookup.keysize() <= 4 {
let path = self.path.join("0.db");
if !path.exists() {
self.create_new_db_file(0)?;
}
return Ok(0);
}
// For keysize 6, we can use multiple files
let path = self.path.join(format!("{}.db", self.last_used_file_nr));
if !path.exists() {
self.create_new_db_file(self.last_used_file_nr)?;
return Ok(self.last_used_file_nr);
}
let metadata = fs::metadata(&path)?;
if metadata.len() >= self.file_size as u64 {
self.last_used_file_nr += 1;
self.create_new_db_file(self.last_used_file_nr)?;
}
Ok(self.last_used_file_nr)
}
/// Stores data at the specified ID with history tracking
pub(crate) fn set_(
&mut self,
id: u32,
old_location: Location,
data: &[u8],
) -> Result<(), Error> {
// Validate data size - maximum is u16::MAX (65535 bytes or ~64KB)
if data.len() > u16::MAX as usize {
return Err(Error::InvalidOperation(format!(
"Data size exceeds maximum allowed size of {} bytes",
u16::MAX
)));
}
// Get file number to use
let file_nr = self.get_file_nr()?;
// Select the file
self.db_file_select(file_nr)?;
// Get current file position for lookup
let file = self
.file
.as_mut()
.ok_or_else(|| Error::Other("No file open".to_string()))?;
file.seek(SeekFrom::End(0))?;
let position = file.stream_position()? as u32;
// Create new location
let new_location = Location { file_nr, position };
// Calculate CRC of data
let crc = calculate_crc(data);
// Create header
let mut header = vec![0u8; HEADER_SIZE];
// Write size (2 bytes)
let size = data.len() as u16; // Safe now because we've validated the size
header[0] = (size & 0xFF) as u8;
header[1] = ((size >> 8) & 0xFF) as u8;
// Write CRC (4 bytes)
header[2] = (crc & 0xFF) as u8;
header[3] = ((crc >> 8) & 0xFF) as u8;
header[4] = ((crc >> 16) & 0xFF) as u8;
header[5] = ((crc >> 24) & 0xFF) as u8;
// Write previous location (6 bytes)
let prev_bytes = old_location.to_bytes();
for (i, &byte) in prev_bytes.iter().enumerate().take(6) {
header[6 + i] = byte;
}
// Write header
file.write_all(&header)?;
// Write actual data
file.write_all(data)?;
file.flush()?;
// Update lookup table with new position
self.lookup.set(id, new_location)?;
Ok(())
}
/// Retrieves data at the specified location
pub(crate) fn get_(&mut self, location: Location) -> Result<Vec<u8>, Error> {
if location.position == 0 {
return Err(Error::NotFound(format!(
"Record not found, location: {:?}",
location
)));
}
// Select the file
self.db_file_select(location.file_nr)?;
let file = self
.file
.as_mut()
.ok_or_else(|| Error::Other("No file open".to_string()))?;
// Read header
file.seek(SeekFrom::Start(location.position as u64))?;
let mut header = vec![0u8; HEADER_SIZE];
file.read_exact(&mut header)?;
// Parse size (2 bytes)
let size = u16::from(header[0]) | (u16::from(header[1]) << 8);
// Parse CRC (4 bytes)
let stored_crc = u32::from(header[2])
| (u32::from(header[3]) << 8)
| (u32::from(header[4]) << 16)
| (u32::from(header[5]) << 24);
// Read data
let mut data = vec![0u8; size as usize];
file.read_exact(&mut data)?;
// Verify CRC
let calculated_crc = calculate_crc(&data);
if calculated_crc != stored_crc {
return Err(Error::DataCorruption(
"CRC mismatch: data corruption detected".to_string(),
));
}
Ok(data)
}
/// Retrieves the previous position for a record (for history tracking)
pub(crate) fn get_prev_pos_(&mut self, location: Location) -> Result<Location, Error> {
if location.position == 0 {
return Err(Error::NotFound("Record not found".to_string()));
}
// Select the file
self.db_file_select(location.file_nr)?;
let file = self
.file
.as_mut()
.ok_or_else(|| Error::Other("No file open".to_string()))?;
// Skip size and CRC (6 bytes)
file.seek(SeekFrom::Start(location.position as u64 + 6))?;
// Read previous location (6 bytes)
let mut prev_bytes = vec![0u8; 6];
file.read_exact(&mut prev_bytes)?;
// Create location from bytes
Location::from_bytes(&prev_bytes, 6)
}
/// Deletes the record at the specified location
pub(crate) fn delete_(&mut self, id: u32, location: Location) -> Result<(), Error> {
if location.position == 0 {
return Err(Error::NotFound("Record not found".to_string()));
}
// Select the file
self.db_file_select(location.file_nr)?;
let file = self
.file
.as_mut()
.ok_or_else(|| Error::Other("No file open".to_string()))?;
// Read size first
file.seek(SeekFrom::Start(location.position as u64))?;
let mut size_bytes = vec![0u8; 2];
file.read_exact(&mut size_bytes)?;
let size = u16::from(size_bytes[0]) | (u16::from(size_bytes[1]) << 8);
// Write zeros for the entire record (header + data)
let zeros = vec![0u8; HEADER_SIZE + size as usize];
file.seek(SeekFrom::Start(location.position as u64))?;
file.write_all(&zeros)?;
// Clear lookup entry
self.lookup.delete(id)?;
Ok(())
}
/// Condenses the database by removing empty records and updating positions
pub fn condense(&mut self) -> Result<(), Error> {
// Create a temporary directory
let temp_path = self.path.join("temp");
fs::create_dir_all(&temp_path)?;
// Get all file numbers
let mut file_numbers = Vec::new();
for entry in fs::read_dir(&self.path)? {
let entry = entry?;
let path = entry.path();
if path.is_file() && path.extension().map_or(false, |ext| ext == "db") {
if let Some(stem) = path.file_stem() {
if let Ok(file_nr) = stem.to_string_lossy().parse::<u16>() {
file_numbers.push(file_nr);
}
}
}
}
// Process each file
for file_nr in file_numbers {
let src_path = self.path.join(format!("{}.db", file_nr));
let temp_file_path = temp_path.join(format!("{}.db", file_nr));
// Create new file
let mut temp_file = File::create(&temp_file_path)?;
temp_file.write_all(&[0u8])?; // Initialize with a byte
// Open source file
let mut src_file = File::open(&src_path)?;
// Read and process records
let mut buffer = vec![0u8; 1024]; // Read in chunks
let mut _position = 0;
while let Ok(bytes_read) = src_file.read(&mut buffer) {
if bytes_read == 0 {
break;
}
// Process the chunk
// This is a simplified version - in a real implementation,
// you would need to handle records that span chunk boundaries
_position += bytes_read;
}
// TODO: Implement proper record copying and position updating
// This would involve:
// 1. Reading each record from the source file
// 2. If not deleted (all zeros), copy to temp file
// 3. Update lookup table with new positions
}
// TODO: Replace original files with temp files
// Clean up
fs::remove_dir_all(&temp_path)?;
Ok(())
}
}
/// Calculates CRC32 for the data
fn calculate_crc(data: &[u8]) -> u32 {
let mut hasher = Hasher::new();
hasher.update(data);
hasher.finalize()
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use crate::{OurDB, OurDBConfig, OurDBSetArgs};
use std::env::temp_dir;
use std::time::{SystemTime, UNIX_EPOCH};
fn get_temp_dir() -> PathBuf {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
temp_dir().join(format!("ourdb_backend_test_{}", timestamp))
}
#[test]
fn test_backend_operations() {
let temp_dir = get_temp_dir();
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: false,
file_size: None,
keysize: None,
reset: None, // Don't reset existing database
};
let mut db = OurDB::new(config).unwrap();
// Test set and get
let test_data = b"Test data for backend operations";
let id = 1;
db.set(OurDBSetArgs {
id: Some(id),
data: test_data,
})
.unwrap();
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved, test_data);
// Clean up
db.destroy().unwrap();
}
}

41
ourdb/src/error.rs
View File

@@ -1,41 +0,0 @@
use thiserror::Error;
/// Error types for OurDB operations
#[derive(Error, Debug)]
pub enum Error {
/// IO errors from file operations
#[error("IO error: {0}")]
Io(#[from] std::io::Error),
/// Data corruption errors
#[error("Data corruption: {0}")]
DataCorruption(String),
/// Invalid operation errors
#[error("Invalid operation: {0}")]
InvalidOperation(String),
/// Lookup table errors
#[error("Lookup error: {0}")]
LookupError(String),
/// Record not found errors
#[error("Record not found: {0}")]
NotFound(String),
/// Other errors
#[error("Error: {0}")]
Other(String),
}
impl From<String> for Error {
fn from(msg: String) -> Self {
Error::Other(msg)
}
}
impl From<&str> for Error {
fn from(msg: &str) -> Self {
Error::Other(msg.to_string())
}
}

293
ourdb/src/lib.rs
View File

@@ -1,293 +0,0 @@
mod backend;
mod error;
mod location;
mod lookup;
pub use error::Error;
pub use location::Location;
pub use lookup::LookupTable;
use std::fs::File;
use std::path::PathBuf;
/// OurDB is a lightweight, efficient key-value database implementation that provides
/// data persistence with history tracking capabilities.
pub struct OurDB {
/// Directory path for storage
path: PathBuf,
/// Whether to use auto-increment mode
incremental_mode: bool,
/// Maximum file size (default: 500MB)
file_size: u32,
/// Lookup table for mapping keys to locations
lookup: LookupTable,
/// Currently open file
file: Option<File>,
/// Current file number
file_nr: u16,
/// Last used file number
last_used_file_nr: u16,
}
/// Configuration for creating a new OurDB instance
pub struct OurDBConfig {
/// Directory path for storage
pub path: PathBuf,
/// Whether to use auto-increment mode
pub incremental_mode: bool,
/// Maximum file size (default: 500MB)
pub file_size: Option<u32>,
/// Lookup table key size (default: 4)
/// - 2: For databases with < 65,536 records (single file)
/// - 3: For databases with < 16,777,216 records (single file)
/// - 4: For databases with < 4,294,967,296 records (single file)
/// - 6: For large databases requiring multiple files (default)
pub keysize: Option<u8>,
/// Whether to reset the database if it exists (default: false)
pub reset: Option<bool>,
}
/// Arguments for setting a value in OurDB
pub struct OurDBSetArgs<'a> {
/// ID for the record (optional in incremental mode)
pub id: Option<u32>,
/// Data to store
pub data: &'a [u8],
}
impl OurDB {
/// Creates a new OurDB instance with the given configuration
pub fn new(config: OurDBConfig) -> Result<Self, Error> {
// If reset is true and the path exists, remove it first
if config.reset.unwrap_or(false) && config.path.exists() {
std::fs::remove_dir_all(&config.path)?;
}
// Create directory if it doesn't exist
std::fs::create_dir_all(&config.path)?;
// Create lookup table
let lookup_path = config.path.join("lookup");
std::fs::create_dir_all(&lookup_path)?;
let lookup_config = lookup::LookupConfig {
size: 1000000, // Default size
keysize: config.keysize.unwrap_or(4),
lookuppath: lookup_path.to_string_lossy().to_string(),
incremental_mode: config.incremental_mode,
};
let lookup = LookupTable::new(lookup_config)?;
let mut db = OurDB {
path: config.path,
incremental_mode: config.incremental_mode,
file_size: config.file_size.unwrap_or(500 * (1 << 20)), // 500MB default
lookup,
file: None,
file_nr: 0,
last_used_file_nr: 0,
};
// Load existing metadata if available
db.load()?;
Ok(db)
}
/// Sets a value in the database
///
/// In incremental mode:
/// - If ID is provided, it updates an existing record
/// - If ID is not provided, it creates a new record with auto-generated ID
///
/// In key-value mode:
/// - ID must be provided
pub fn set(&mut self, args: OurDBSetArgs) -> Result<u32, Error> {
if self.incremental_mode {
if let Some(id) = args.id {
// This is an update
let location = self.lookup.get(id)?;
if location.position == 0 {
return Err(Error::InvalidOperation(
"Cannot set ID for insertions when incremental mode is enabled".to_string(),
));
}
self.set_(id, location, args.data)?;
Ok(id)
} else {
// This is an insert
let id = self.lookup.get_next_id()?;
self.set_(id, Location::default(), args.data)?;
Ok(id)
}
} else {
// Using key-value mode
let id = args.id.ok_or_else(|| {
Error::InvalidOperation(
"ID must be provided when incremental is disabled".to_string(),
)
})?;
let location = self.lookup.get(id)?;
self.set_(id, location, args.data)?;
Ok(id)
}
}
/// Retrieves data stored at the specified key position
pub fn get(&mut self, id: u32) -> Result<Vec<u8>, Error> {
let location = self.lookup.get(id)?;
self.get_(location)
}
/// Retrieves a list of previous values for the specified key
///
/// The depth parameter controls how many historical values to retrieve (maximum)
pub fn get_history(&mut self, id: u32, depth: u8) -> Result<Vec<Vec<u8>>, Error> {
let mut result = Vec::new();
let mut current_location = self.lookup.get(id)?;
// Traverse the history chain up to specified depth
for _ in 0..depth {
// Get current value
let data = self.get_(current_location)?;
result.push(data);
// Try to get previous location
match self.get_prev_pos_(current_location) {
Ok(location) => {
if location.position == 0 {
break;
}
current_location = location;
}
Err(_) => break,
}
}
Ok(result)
}
/// Deletes the data at the specified key position
pub fn delete(&mut self, id: u32) -> Result<(), Error> {
let location = self.lookup.get(id)?;
self.delete_(id, location)?;
self.lookup.delete(id)?;
Ok(())
}
/// Returns the next ID which will be used when storing in incremental mode
pub fn get_next_id(&mut self) -> Result<u32, Error> {
if !self.incremental_mode {
return Err(Error::InvalidOperation(
"Incremental mode is not enabled".to_string(),
));
}
self.lookup.get_next_id()
}
/// Closes the database, ensuring all data is saved
pub fn close(&mut self) -> Result<(), Error> {
self.save()?;
self.close_();
Ok(())
}
/// Destroys the database, removing all files
pub fn destroy(&mut self) -> Result<(), Error> {
let _ = self.close();
std::fs::remove_dir_all(&self.path)?;
Ok(())
}
// Helper methods
fn lookup_dump_path(&self) -> PathBuf {
self.path.join("lookup_dump.db")
}
fn load(&mut self) -> Result<(), Error> {
let dump_path = self.lookup_dump_path();
if dump_path.exists() {
self.lookup.import_sparse(&dump_path.to_string_lossy())?;
}
Ok(())
}
fn save(&mut self) -> Result<(), Error> {
self.lookup
.export_sparse(&self.lookup_dump_path().to_string_lossy())?;
Ok(())
}
fn close_(&mut self) {
self.file = None;
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::env::temp_dir;
use std::time::{SystemTime, UNIX_EPOCH};
fn get_temp_dir() -> PathBuf {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
temp_dir().join(format!("ourdb_test_{}", timestamp))
}
#[test]
fn test_basic_operations() {
let temp_dir = get_temp_dir();
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: None, // Don't reset existing database
};
let mut db = OurDB::new(config).unwrap();
// Test set and get
let test_data = b"Hello, OurDB!";
let id = db
.set(OurDBSetArgs {
id: None,
data: test_data,
})
.unwrap();
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved, test_data);
// Test update
let updated_data = b"Updated data";
db.set(OurDBSetArgs {
id: Some(id),
data: updated_data,
})
.unwrap();
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved, updated_data);
// Test history
let history = db.get_history(id, 2).unwrap();
assert_eq!(history.len(), 2);
assert_eq!(history[0], updated_data);
assert_eq!(history[1], test_data);
// Test delete
db.delete(id).unwrap();
assert!(db.get(id).is_err());
// Clean up
db.destroy().unwrap();
}
}

178
ourdb/src/location.rs
View File

@@ -1,178 +0,0 @@
use crate::error::Error;
/// Location represents a physical position in a database file
///
/// It consists of a file number and a position within that file.
/// This allows OurDB to span multiple files for large datasets.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Location {
/// File number (0-65535)
pub file_nr: u16,
/// Position within the file
pub position: u32,
}
impl Location {
/// Creates a new Location from bytes based on keysize
///
/// - keysize = 2: Only position (2 bytes), file_nr = 0
/// - keysize = 3: Only position (3 bytes), file_nr = 0
/// - keysize = 4: Only position (4 bytes), file_nr = 0
/// - keysize = 6: file_nr (2 bytes) + position (4 bytes)
pub fn from_bytes(bytes: &[u8], keysize: u8) -> Result<Self, Error> {
// Validate keysize
if ![2, 3, 4, 6].contains(&keysize) {
return Err(Error::InvalidOperation(format!(
"Invalid keysize: {}",
keysize
)));
}
// Create padded bytes
let mut padded = vec![0u8; keysize as usize];
if bytes.len() > keysize as usize {
return Err(Error::InvalidOperation(
"Input bytes exceed keysize".to_string(),
));
}
let start_idx = keysize as usize - bytes.len();
for (i, &b) in bytes.iter().enumerate() {
if i + start_idx < padded.len() {
padded[start_idx + i] = b;
}
}
let mut location = Location::default();
match keysize {
2 => {
// Only position, 2 bytes big endian
location.position = u32::from(padded[0]) << 8 | u32::from(padded[1]);
location.file_nr = 0;
// Verify limits
if location.position > 0xFFFF {
return Err(Error::InvalidOperation(
"Position exceeds max value for keysize=2 (max 65535)".to_string(),
));
}
}
3 => {
// Only position, 3 bytes big endian
location.position =
u32::from(padded[0]) << 16 | u32::from(padded[1]) << 8 | u32::from(padded[2]);
location.file_nr = 0;
// Verify limits
if location.position > 0xFFFFFF {
return Err(Error::InvalidOperation(
"Position exceeds max value for keysize=3 (max 16777215)".to_string(),
));
}
}
4 => {
// Only position, 4 bytes big endian
location.position = u32::from(padded[0]) << 24
| u32::from(padded[1]) << 16
| u32::from(padded[2]) << 8
| u32::from(padded[3]);
location.file_nr = 0;
}
6 => {
// 2 bytes file_nr + 4 bytes position, all big endian
location.file_nr = u16::from(padded[0]) << 8 | u16::from(padded[1]);
location.position = u32::from(padded[2]) << 24
| u32::from(padded[3]) << 16
| u32::from(padded[4]) << 8
| u32::from(padded[5]);
}
_ => unreachable!(),
}
Ok(location)
}
/// Converts the location to bytes (always 6 bytes)
///
/// Format: [file_nr (2 bytes)][position (4 bytes)]
pub fn to_bytes(&self) -> Vec<u8> {
let mut bytes = Vec::with_capacity(6);
// Put file_nr first (2 bytes)
bytes.push((self.file_nr >> 8) as u8);
bytes.push(self.file_nr as u8);
// Put position next (4 bytes)
bytes.push((self.position >> 24) as u8);
bytes.push((self.position >> 16) as u8);
bytes.push((self.position >> 8) as u8);
bytes.push(self.position as u8);
bytes
}
/// Converts the location to a u64 value
///
/// The file_nr is stored in the most significant bits
pub fn to_u64(&self) -> u64 {
(u64::from(self.file_nr) << 32) | u64::from(self.position)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_location_from_bytes_keysize_2() {
let bytes = vec![0x12, 0x34];
let location = Location::from_bytes(&bytes, 2).unwrap();
assert_eq!(location.file_nr, 0);
assert_eq!(location.position, 0x1234);
}
#[test]
fn test_location_from_bytes_keysize_3() {
let bytes = vec![0x12, 0x34, 0x56];
let location = Location::from_bytes(&bytes, 3).unwrap();
assert_eq!(location.file_nr, 0);
assert_eq!(location.position, 0x123456);
}
#[test]
fn test_location_from_bytes_keysize_4() {
let bytes = vec![0x12, 0x34, 0x56, 0x78];
let location = Location::from_bytes(&bytes, 4).unwrap();
assert_eq!(location.file_nr, 0);
assert_eq!(location.position, 0x12345678);
}
#[test]
fn test_location_from_bytes_keysize_6() {
let bytes = vec![0xAB, 0xCD, 0x12, 0x34, 0x56, 0x78];
let location = Location::from_bytes(&bytes, 6).unwrap();
assert_eq!(location.file_nr, 0xABCD);
assert_eq!(location.position, 0x12345678);
}
#[test]
fn test_location_to_bytes() {
let location = Location {
file_nr: 0xABCD,
position: 0x12345678,
};
let bytes = location.to_bytes();
assert_eq!(bytes, vec![0xAB, 0xCD, 0x12, 0x34, 0x56, 0x78]);
}
#[test]
fn test_location_to_u64() {
let location = Location {
file_nr: 0xABCD,
position: 0x12345678,
};
let value = location.to_u64();
assert_eq!(value, 0xABCD_0000_0000 | 0x12345678);
}
}

540
ourdb/src/lookup.rs
View File

@@ -1,540 +0,0 @@
use std::fs::{self, File, OpenOptions};
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::Path;
use crate::error::Error;
use crate::location::Location;
const DATA_FILE_NAME: &str = "data";
const INCREMENTAL_FILE_NAME: &str = ".inc";
/// Configuration for creating a new lookup table
pub struct LookupConfig {
/// Size of the lookup table
pub size: u32,
/// Size of each entry in bytes (2-6)
/// - 2: For databases with < 65,536 records (single file)
/// - 3: For databases with < 16,777,216 records (single file)
/// - 4: For databases with < 4,294,967,296 records (single file)
/// - 6: For large databases requiring multiple files
pub keysize: u8,
/// Path for disk-based lookup
pub lookuppath: String,
/// Whether to use incremental mode
pub incremental_mode: bool,
}
/// Lookup table maps keys to physical locations in the backend storage
pub struct LookupTable {
/// Size of each entry in bytes (2-6)
keysize: u8,
/// Path for disk-based lookup
lookuppath: String,
/// In-memory data for memory-based lookup
data: Vec<u8>,
/// Next empty slot if incremental mode is enabled
incremental: Option<u32>,
}
impl LookupTable {
/// Returns the keysize of this lookup table
pub fn keysize(&self) -> u8 {
self.keysize
}
/// Creates a new lookup table with the given configuration
pub fn new(config: LookupConfig) -> Result<Self, Error> {
// Verify keysize is valid
if ![2, 3, 4, 6].contains(&config.keysize) {
return Err(Error::InvalidOperation(format!(
"Invalid keysize: {}",
config.keysize
)));
}
let incremental = if config.incremental_mode {
Some(get_incremental_info(&config)?)
} else {
None
};
if !config.lookuppath.is_empty() {
// Create directory if it doesn't exist
fs::create_dir_all(&config.lookuppath)?;
// For disk-based lookup, create empty file if it doesn't exist
let data_path = Path::new(&config.lookuppath).join(DATA_FILE_NAME);
if !data_path.exists() {
let data = vec![0u8; config.size as usize * config.keysize as usize];
fs::write(&data_path, &data)?;
}
Ok(LookupTable {
data: Vec::new(),
keysize: config.keysize,
lookuppath: config.lookuppath,
incremental,
})
} else {
// For memory-based lookup
Ok(LookupTable {
data: vec![0u8; config.size as usize * config.keysize as usize],
keysize: config.keysize,
lookuppath: String::new(),
incremental,
})
}
}
/// Gets a location for the given ID
pub fn get(&self, id: u32) -> Result<Location, Error> {
let entry_size = self.keysize as usize;
if !self.lookuppath.is_empty() {
// Disk-based lookup
let data_path = Path::new(&self.lookuppath).join(DATA_FILE_NAME);
// Check file size first
let file_size = fs::metadata(&data_path)?.len();
let start_pos = id as u64 * entry_size as u64;
if start_pos + entry_size as u64 > file_size {
return Err(Error::LookupError(format!(
"Invalid read for get in lut: {}: {} would exceed file size {}",
self.lookuppath,
start_pos + entry_size as u64,
file_size
)));
}
// Read directly from file
let mut file = File::open(&data_path)?;
file.seek(SeekFrom::Start(start_pos))?;
let mut data = vec![0u8; entry_size];
let bytes_read = file.read(&mut data)?;
if bytes_read < entry_size {
return Err(Error::LookupError(format!(
"Incomplete read: expected {} bytes but got {}",
entry_size, bytes_read
)));
}
return Location::from_bytes(&data, self.keysize);
}
// Memory-based lookup
if (id * self.keysize as u32) as usize >= self.data.len() {
return Err(Error::LookupError("Index out of bounds".to_string()));
}
let start = (id * self.keysize as u32) as usize;
let end = start + entry_size;
Location::from_bytes(&self.data[start..end], self.keysize)
}
/// Sets a location for the given ID
pub fn set(&mut self, id: u32, location: Location) -> Result<(), Error> {
let entry_size = self.keysize as usize;
// Handle incremental mode
if let Some(incremental) = self.incremental {
if id == incremental {
self.increment_index()?;
}
if id > incremental {
return Err(Error::InvalidOperation(
"Cannot set ID for insertions when incremental mode is enabled".to_string(),
));
}
}
// Convert location to bytes based on keysize
let location_bytes = match self.keysize {
2 => {
if location.file_nr != 0 {
return Err(Error::InvalidOperation(
"file_nr must be 0 for keysize=2".to_string(),
));
}
if location.position > 0xFFFF {
return Err(Error::InvalidOperation(
"position exceeds max value for keysize=2 (max 65535)".to_string(),
));
}
vec![(location.position >> 8) as u8, location.position as u8]
}
3 => {
if location.file_nr != 0 {
return Err(Error::InvalidOperation(
"file_nr must be 0 for keysize=3".to_string(),
));
}
if location.position > 0xFFFFFF {
return Err(Error::InvalidOperation(
"position exceeds max value for keysize=3 (max 16777215)".to_string(),
));
}
vec![
(location.position >> 16) as u8,
(location.position >> 8) as u8,
location.position as u8,
]
}
4 => {
if location.file_nr != 0 {
return Err(Error::InvalidOperation(
"file_nr must be 0 for keysize=4".to_string(),
));
}
vec![
(location.position >> 24) as u8,
(location.position >> 16) as u8,
(location.position >> 8) as u8,
location.position as u8,
]
}
6 => {
// Full location with file_nr and position
location.to_bytes()
}
_ => {
return Err(Error::InvalidOperation(format!(
"Invalid keysize: {}",
self.keysize
)))
}
};
if !self.lookuppath.is_empty() {
// Disk-based lookup
let data_path = Path::new(&self.lookuppath).join(DATA_FILE_NAME);
let mut file = OpenOptions::new().write(true).open(data_path)?;
let start_pos = id as u64 * entry_size as u64;
file.seek(SeekFrom::Start(start_pos))?;
file.write_all(&location_bytes)?;
} else {
// Memory-based lookup
let start = (id * self.keysize as u32) as usize;
if start + entry_size > self.data.len() {
return Err(Error::LookupError("Index out of bounds".to_string()));
}
for (i, &byte) in location_bytes.iter().enumerate() {
self.data[start + i] = byte;
}
}
Ok(())
}
/// Deletes an entry for the given ID
pub fn delete(&mut self, id: u32) -> Result<(), Error> {
// Set location to all zeros
self.set(id, Location::default())
}
/// Gets the next available ID in incremental mode
pub fn get_next_id(&self) -> Result<u32, Error> {
let incremental = self.incremental.ok_or_else(|| {
Error::InvalidOperation("Lookup table not in incremental mode".to_string())
})?;
let table_size = if !self.lookuppath.is_empty() {
let data_path = Path::new(&self.lookuppath).join(DATA_FILE_NAME);
fs::metadata(data_path)?.len() as u32
} else {
self.data.len() as u32
};
if incremental * self.keysize as u32 >= table_size {
return Err(Error::LookupError("Lookup table is full".to_string()));
}
Ok(incremental)
}
/// Increments the index in incremental mode
pub fn increment_index(&mut self) -> Result<(), Error> {
let mut incremental = self.incremental.ok_or_else(|| {
Error::InvalidOperation("Lookup table not in incremental mode".to_string())
})?;
incremental += 1;
self.incremental = Some(incremental);
if !self.lookuppath.is_empty() {
let inc_path = Path::new(&self.lookuppath).join(INCREMENTAL_FILE_NAME);
fs::write(inc_path, incremental.to_string())?;
}
Ok(())
}
/// Exports the lookup table to a file
pub fn export_data(&self, path: &str) -> Result<(), Error> {
if !self.lookuppath.is_empty() {
// For disk-based lookup, just copy the file
let data_path = Path::new(&self.lookuppath).join(DATA_FILE_NAME);
fs::copy(data_path, path)?;
} else {
// For memory-based lookup, write the data to file
fs::write(path, &self.data)?;
}
Ok(())
}
/// Imports the lookup table from a file
pub fn import_data(&mut self, path: &str) -> Result<(), Error> {
if !self.lookuppath.is_empty() {
// For disk-based lookup, copy the file
let data_path = Path::new(&self.lookuppath).join(DATA_FILE_NAME);
fs::copy(path, data_path)?;
} else {
// For memory-based lookup, read the data from file
self.data = fs::read(path)?;
}
Ok(())
}
/// Exports only non-zero entries to save space
pub fn export_sparse(&self, path: &str) -> Result<(), Error> {
let mut output = Vec::new();
let entry_size = self.keysize as usize;
if !self.lookuppath.is_empty() {
// For disk-based lookup
let data_path = Path::new(&self.lookuppath).join(DATA_FILE_NAME);
let mut file = File::open(&data_path)?;
let file_size = fs::metadata(&data_path)?.len();
let max_entries = file_size / entry_size as u64;
for id in 0..max_entries {
file.seek(SeekFrom::Start(id * entry_size as u64))?;
let mut buffer = vec![0u8; entry_size];
let bytes_read = file.read(&mut buffer)?;
if bytes_read < entry_size {
break;
}
// Check if entry is non-zero
if buffer.iter().any(|&b| b != 0) {
// Write ID (4 bytes) + entry
output.extend_from_slice(&(id as u32).to_be_bytes());
output.extend_from_slice(&buffer);
}
}
} else {
// For memory-based lookup
let max_entries = self.data.len() / entry_size;
for id in 0..max_entries {
let start = id * entry_size;
let entry = &self.data[start..start + entry_size];
// Check if entry is non-zero
if entry.iter().any(|&b| b != 0) {
// Write ID (4 bytes) + entry
output.extend_from_slice(&(id as u32).to_be_bytes());
output.extend_from_slice(entry);
}
}
}
// Write the output to file
fs::write(path, &output)?;
Ok(())
}
/// Imports sparse data (only non-zero entries)
pub fn import_sparse(&mut self, path: &str) -> Result<(), Error> {
let data = fs::read(path)?;
let entry_size = self.keysize as usize;
let record_size = 4 + entry_size; // ID (4 bytes) + entry
if data.len() % record_size != 0 {
return Err(Error::DataCorruption(
"Invalid sparse data format: size mismatch".to_string(),
));
}
for chunk_start in (0..data.len()).step_by(record_size) {
if chunk_start + record_size > data.len() {
break;
}
// Extract ID (4 bytes)
let id_bytes = &data[chunk_start..chunk_start + 4];
let id = u32::from_be_bytes([id_bytes[0], id_bytes[1], id_bytes[2], id_bytes[3]]);
// Extract entry
let entry = &data[chunk_start + 4..chunk_start + record_size];
// Create location from entry
let location = Location::from_bytes(entry, self.keysize)?;
// Set the entry
self.set(id, location)?;
}
Ok(())
}
/// Finds the highest ID with a non-zero entry
pub fn find_last_entry(&mut self) -> Result<u32, Error> {
let mut last_id = 0u32;
let entry_size = self.keysize as usize;
if !self.lookuppath.is_empty() {
// For disk-based lookup
let data_path = Path::new(&self.lookuppath).join(DATA_FILE_NAME);
let mut file = File::open(&data_path)?;
let file_size = fs::metadata(&data_path)?.len();
let mut buffer = vec![0u8; entry_size];
let mut pos = 0u32;
while (pos as u64 * entry_size as u64) < file_size {
file.seek(SeekFrom::Start(pos as u64 * entry_size as u64))?;
let bytes_read = file.read(&mut buffer)?;
if bytes_read == 0 || bytes_read < entry_size {
break;
}
let location = Location::from_bytes(&buffer, self.keysize)?;
if location.position != 0 || location.file_nr != 0 {
last_id = pos;
}
pos += 1;
}
} else {
// For memory-based lookup
for i in 0..(self.data.len() / entry_size) as u32 {
if let Ok(location) = self.get(i) {
if location.position != 0 || location.file_nr != 0 {
last_id = i;
}
}
}
}
Ok(last_id)
}
}
/// Helper function to get the incremental value
fn get_incremental_info(config: &LookupConfig) -> Result<u32, Error> {
if !config.incremental_mode {
return Ok(0);
}
if !config.lookuppath.is_empty() {
let inc_path = Path::new(&config.lookuppath).join(INCREMENTAL_FILE_NAME);
if !inc_path.exists() {
// Create a separate file for storing the incremental value
fs::write(&inc_path, "1")?;
}
let inc_str = fs::read_to_string(&inc_path)?;
let incremental = match inc_str.trim().parse::<u32>() {
Ok(val) => val,
Err(_) => {
// If the value is invalid, reset it to 1
fs::write(&inc_path, "1")?;
1
}
};
Ok(incremental)
} else {
// For memory-based lookup, start with 1
Ok(1)
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::env::temp_dir;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
fn get_temp_dir() -> PathBuf {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_secs();
temp_dir().join(format!("ourdb_lookup_test_{}", timestamp))
}
#[test]
fn test_memory_lookup() {
let config = LookupConfig {
size: 1000,
keysize: 4,
lookuppath: String::new(),
incremental_mode: true,
};
let mut lookup = LookupTable::new(config).unwrap();
// Test set and get
let location = Location {
file_nr: 0,
position: 12345,
};
lookup.set(1, location).unwrap();
let retrieved = lookup.get(1).unwrap();
assert_eq!(retrieved.file_nr, location.file_nr);
assert_eq!(retrieved.position, location.position);
// Test incremental mode
let next_id = lookup.get_next_id().unwrap();
assert_eq!(next_id, 2);
lookup.increment_index().unwrap();
let next_id = lookup.get_next_id().unwrap();
assert_eq!(next_id, 3);
}
#[test]
fn test_disk_lookup() {
let temp_dir = get_temp_dir();
fs::create_dir_all(&temp_dir).unwrap();
let config = LookupConfig {
size: 1000,
keysize: 4,
lookuppath: temp_dir.to_string_lossy().to_string(),
incremental_mode: true,
};
let mut lookup = LookupTable::new(config).unwrap();
// Test set and get
let location = Location {
file_nr: 0,
position: 12345,
};
lookup.set(1, location).unwrap();
let retrieved = lookup.get(1).unwrap();
assert_eq!(retrieved.file_nr, location.file_nr);
assert_eq!(retrieved.position, location.position);
// Clean up
fs::remove_dir_all(temp_dir).unwrap();
}
}

369
ourdb/tests/integration_tests.rs
View File

@@ -1,369 +0,0 @@
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use rand;
use std::env::temp_dir;
use std::fs;
use std::path::PathBuf;
use std::time::{SystemTime, UNIX_EPOCH};
// Helper function to create a unique temporary directory for tests
fn get_temp_dir() -> PathBuf {
let timestamp = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos();
let random_part = rand::random::<u32>();
let dir = temp_dir().join(format!("ourdb_test_{}_{}", timestamp, random_part));
// Ensure the directory exists and is empty
if dir.exists() {
std::fs::remove_dir_all(&dir).unwrap();
}
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn test_basic_operations() {
let temp_dir = get_temp_dir();
// Create a new database with incremental mode
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: None,
};
let mut db = OurDB::new(config).unwrap();
// Test set and get
let test_data = b"Hello, OurDB!";
let id = db
.set(OurDBSetArgs {
id: None,
data: test_data,
})
.unwrap();
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved, test_data);
// Test update
let updated_data = b"Updated data";
db.set(OurDBSetArgs {
id: Some(id),
data: updated_data,
})
.unwrap();
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved, updated_data);
// Test history
let history = db.get_history(id, 2).unwrap();
assert_eq!(history.len(), 2);
assert_eq!(history[0], updated_data);
assert_eq!(history[1], test_data);
// Test delete
db.delete(id).unwrap();
assert!(db.get(id).is_err());
// Clean up
db.destroy().unwrap();
}
#[test]
fn test_key_value_mode() {
let temp_dir = get_temp_dir();
// Create a new database with key-value mode
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: false,
file_size: None,
keysize: None,
reset: None,
};
let mut db = OurDB::new(config).unwrap();
// Test set with explicit ID
let test_data = b"Key-value data";
let id = 42;
db.set(OurDBSetArgs {
id: Some(id),
data: test_data,
})
.unwrap();
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved, test_data);
// Verify next_id fails in key-value mode
assert!(db.get_next_id().is_err());
// Clean up
db.destroy().unwrap();
}
#[test]
fn test_incremental_mode() {
let temp_dir = get_temp_dir();
// Create a new database with incremental mode
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: None,
};
let mut db = OurDB::new(config).unwrap();
// Test auto-increment IDs
let data1 = b"First record";
let id1 = db
.set(OurDBSetArgs {
id: None,
data: data1,
})
.unwrap();
let data2 = b"Second record";
let id2 = db
.set(OurDBSetArgs {
id: None,
data: data2,
})
.unwrap();
// IDs should be sequential
assert_eq!(id2, id1 + 1);
// Verify get_next_id works
let next_id = db.get_next_id().unwrap();
assert_eq!(next_id, id2 + 1);
// Clean up
db.destroy().unwrap();
}
#[test]
fn test_persistence() {
let temp_dir = get_temp_dir();
// Create data in a new database
{
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: None,
};
let mut db = OurDB::new(config).unwrap();
let test_data = b"Persistent data";
let id = db
.set(OurDBSetArgs {
id: None,
data: test_data,
})
.unwrap();
// Explicitly close the database
db.close().unwrap();
// ID should be 1 in a new database
assert_eq!(id, 1);
}
// Reopen the database and verify data persists
{
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: None,
};
let mut db = OurDB::new(config).unwrap();
// Verify data is still there
let retrieved = db.get(1).unwrap();
assert_eq!(retrieved, b"Persistent data");
// Verify incremental counter persisted
let next_id = db.get_next_id().unwrap();
assert_eq!(next_id, 2);
// Clean up
db.destroy().unwrap();
}
}
#[test]
fn test_different_keysizes() {
for keysize in [2, 3, 4, 6].iter() {
let temp_dir = get_temp_dir();
// Ensure the directory exists
std::fs::create_dir_all(&temp_dir).unwrap();
// Create a new database with specified keysize
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: None,
keysize: Some(*keysize),
reset: None,
};
let mut db = OurDB::new(config).unwrap();
// Test basic operations
let test_data = b"Keysize test data";
let id = db
.set(OurDBSetArgs {
id: None,
data: test_data,
})
.unwrap();
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved, test_data);
// Clean up
db.destroy().unwrap();
}
}
#[test]
fn test_large_data() {
let temp_dir = get_temp_dir();
// Create a new database
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: None,
};
let mut db = OurDB::new(config).unwrap();
// Create a large data set (60KB - within the 64KB limit)
let large_data = vec![b'X'; 60 * 1024];
// Store and retrieve large data
let id = db
.set(OurDBSetArgs {
id: None,
data: &large_data,
})
.unwrap();
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved.len(), large_data.len());
assert_eq!(retrieved, large_data);
// Clean up
db.destroy().unwrap();
}
#[test]
fn test_exceed_size_limit() {
let temp_dir = get_temp_dir();
// Create a new database
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: None,
keysize: None,
reset: None,
};
let mut db = OurDB::new(config).unwrap();
// Create data larger than the 64KB limit (70KB)
let oversized_data = vec![b'X'; 70 * 1024];
// Attempt to store data that exceeds the size limit
let result = db.set(OurDBSetArgs {
id: None,
data: &oversized_data,
});
// Verify that an error is returned
assert!(
result.is_err(),
"Expected an error when storing data larger than 64KB"
);
// Clean up
db.destroy().unwrap();
}
#[test]
fn test_multiple_files() {
let temp_dir = get_temp_dir();
// Create a new database with small file size to force multiple files
let config = OurDBConfig {
path: temp_dir.clone(),
incremental_mode: true,
file_size: Some(1024), // Very small file size (1KB)
keysize: Some(6), // 6-byte keysize for multiple files
reset: None,
};
let mut db = OurDB::new(config).unwrap();
// Store enough data to span multiple files
let data_size = 500; // bytes per record
let test_data = vec![b'A'; data_size];
let mut ids = Vec::new();
for _ in 0..10 {
let id = db
.set(OurDBSetArgs {
id: None,
data: &test_data,
})
.unwrap();
ids.push(id);
}
// Verify all data can be retrieved
for &id in &ids {
let retrieved = db.get(id).unwrap();
assert_eq!(retrieved.len(), data_size);
}
// Verify multiple files were created
let files = fs::read_dir(&temp_dir)
.unwrap()
.filter_map(Result::ok)
.filter(|entry| {
let path = entry.path();
path.is_file() && path.extension().map_or(false, |ext| ext == "db")
})
.count();
assert!(
files > 1,
"Expected multiple database files, found {}",
files
);
// Clean up
db.destroy().unwrap();
}

787
radixtree/ARCHITECTURE.md
View File

@@ -1,787 +0,0 @@
# RadixTree: Architecture for V to Rust Port
## 1. Overview
RadixTree is a space-optimized tree data structure that enables efficient string key operations with persistent storage. This document outlines the architecture for porting the RadixTree module from its original V implementation to Rust, maintaining all existing functionality while leveraging Rust's memory safety, performance, and ecosystem.
The Rust implementation will integrate with the existing OurDB Rust implementation for persistent storage.
```mermaid
graph TD
A[Client Code] --> B[RadixTree API]
B --> C[Node Management]
B --> D[Serialization]
B --> E[Tree Operations]
C --> F[OurDB]
D --> F
E --> C
```
## 2. Current Architecture (V Implementation)
The current V implementation of RadixTree consists of the following components:
### 2.1 Core Data Structures
#### Node
```v
struct Node {
mut:
key_segment string // The segment of the key stored at this node
value []u8 // Value stored at this node (empty if not a leaf)
children []NodeRef // References to child nodes
is_leaf bool // Whether this node is a leaf node
}
```
#### NodeRef
```v
struct NodeRef {
mut:
key_part string // The key segment for this child
node_id u32 // Database ID of the node
}
```
#### RadixTree
```v
@[heap]
pub struct RadixTree {
mut:
db &ourdb.OurDB // Database for persistent storage
root_id u32 // Database ID of the root node
}
```
### 2.2 Key Operations
1. **new()**: Creates a new radix tree with a specified database path
2. **set(key, value)**: Sets a key-value pair in the tree
3. **get(key)**: Retrieves a value by key
4. **update(prefix, new_value)**: Updates the value at a given key prefix
5. **delete(key)**: Removes a key from the tree
6. **list(prefix)**: Lists all keys with a given prefix
7. **getall(prefix)**: Gets all values for keys with a given prefix
### 2.3 Serialization
The V implementation uses a custom binary serialization format for nodes:
- Version byte (1 byte)
- Key segment (string)
- Value length (2 bytes) followed by value bytes
- Children count (2 bytes) followed by children
- Is leaf flag (1 byte)
Each child is serialized as:
- Key part (string)
- Node ID (4 bytes)
### 2.4 Integration with OurDB
The RadixTree uses OurDB for persistent storage:
- Each node is serialized and stored as a record in OurDB
- Node references use OurDB record IDs
- The tree maintains a root node ID for traversal
## 3. Proposed Rust Architecture
The Rust implementation will maintain the same overall architecture while leveraging Rust's type system, ownership model, and error handling.
### 3.1 Core Data Structures
#### Node
```rust
pub struct Node {
key_segment: String,
value: Vec<u8>,
children: Vec<NodeRef>,
is_leaf: bool,
}
```
#### NodeRef
```rust
pub struct NodeRef {
key_part: String,
node_id: u32,
}
```
#### RadixTree
```rust
pub struct RadixTree {
db: ourdb::OurDB,
root_id: u32,
}
```
### 3.2 Public API
```rust
impl RadixTree {
/// Creates a new radix tree with the specified database path
pub fn new(path: &str, reset: bool) -> Result<Self, Error> {
// Implementation
}
/// Sets a key-value pair in the tree
pub fn set(&mut self, key: &str, value: Vec<u8>) -> Result<(), Error> {
// Implementation
}
/// Gets a value by key from the tree
pub fn get(&mut self, key: &str) -> Result<Vec<u8>, Error> {
// Implementation
}
/// Updates the value at a given key prefix
pub fn update(&mut self, prefix: &str, new_value: Vec<u8>) -> Result<(), Error> {
// Implementation
}
/// Deletes a key from the tree
pub fn delete(&mut self, key: &str) -> Result<(), Error> {
// Implementation
}
/// Lists all keys with a given prefix
pub fn list(&mut self, prefix: &str) -> Result<Vec<String>, Error> {
// Implementation
}
/// Gets all values for keys with a given prefix
pub fn getall(&mut self, prefix: &str) -> Result<Vec<Vec<u8>>, Error> {
// Implementation
}
}
```
### 3.3 Error Handling
```rust
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("OurDB error: {0}")]
OurDB(#[from] ourdb::Error),
#[error("Key not found: {0}")]
KeyNotFound(String),
#[error("Prefix not found: {0}")]
PrefixNotFound(String),
#[error("Serialization error: {0}")]
Serialization(String),
#[error("Deserialization error: {0}")]
Deserialization(String),
#[error("Invalid operation: {0}")]
InvalidOperation(String),
}
```
### 3.4 Serialization
The Rust implementation will maintain the same binary serialization format for compatibility:
```rust
const VERSION: u8 = 1;
impl Node {
/// Serializes a node to bytes for storage
fn serialize(&self) -> Vec<u8> {
// Implementation
}
/// Deserializes bytes to a node
fn deserialize(data: &[u8]) -> Result<Self, Error> {
// Implementation
}
}
```
### 3.5 Integration with OurDB
The Rust implementation will use the existing OurDB Rust implementation:
```rust
impl RadixTree {
fn get_node(&mut self, node_id: u32) -> Result<Node, Error> {
let data = self.db.get(node_id)?;
Node::deserialize(&data)
}
fn save_node(&mut self, node_id: Option<u32>, node: &Node) -> Result<u32, Error> {
let data = node.serialize();
let args = ourdb::OurDBSetArgs {
id: node_id,
data: &data,
};
Ok(self.db.set(args)?)
}
}
```
## 4. Implementation Strategy
### 4.1 Phase 1: Core Data Structures and Serialization
1. Implement the `Node` and `NodeRef` structs
2. Implement serialization and deserialization functions
3. Implement the `Error` enum for error handling
### 4.2 Phase 2: Basic Tree Operations
1. Implement the `RadixTree` struct with OurDB integration
2. Implement the `new()` function for creating a new tree
3. Implement the `get()` and `set()` functions for basic operations
### 4.3 Phase 3: Advanced Tree Operations
1. Implement the `delete()` function for removing keys
2. Implement the `update()` function for updating values
3. Implement the `list()` and `getall()` functions for prefix operations
### 4.4 Phase 4: Testing and Optimization
1. Port existing tests from V to Rust
2. Add new tests for Rust-specific functionality
3. Benchmark and optimize performance
4. Ensure compatibility with existing RadixTree data
## 5. Implementation Considerations
### 5.1 Memory Management
Leverage Rust's ownership model for safe and efficient memory management:
- Use `String` and `Vec<u8>` for data buffers instead of raw pointers
- Use references and borrows to avoid unnecessary copying
- Implement proper RAII for resource management
### 5.2 Error Handling
Use Rust's `Result` type for comprehensive error handling:
- Define custom error types for RadixTree-specific errors
- Propagate errors using the `?` operator
- Provide detailed error messages
- Implement proper error conversion using the `From` trait
### 5.3 Performance Optimizations
Identify opportunities for performance improvements:
- Use efficient string operations for prefix matching
- Minimize database operations by caching nodes when appropriate
- Use iterators for efficient traversal
- Consider using `Cow<str>` for string operations to avoid unnecessary cloning
### 5.4 Compatibility
Ensure compatibility with the V implementation:
- Maintain the same serialization format
- Ensure identical behavior for all operations
- Support reading existing RadixTree data
## 6. Testing Strategy
### 6.1 Unit Tests
Write comprehensive unit tests for each component:
- Test `Node` serialization/deserialization
- Test string operations (common prefix, etc.)
- Test error handling
### 6.2 Integration Tests
Write integration tests for the complete system:
- Test basic CRUD operations
- Test prefix operations
- Test edge cases (empty keys, very long keys, etc.)
- Test with large datasets
### 6.3 Compatibility Tests
Ensure compatibility with existing RadixTree data:
- Test reading existing V-created RadixTree data
- Test writing data that can be read by the V implementation
### 6.4 Performance Tests
Benchmark performance against the V implementation:
- Measure throughput for set/get operations
- Measure latency for different operations
- Test with different tree sizes and key distributions
## 7. Project Structure
```
radixtree/
├── Cargo.toml
├── src/
│ ├── lib.rs # Public API and re-exports
│ ├── node.rs # Node and NodeRef implementations
│ ├── serialize.rs # Serialization and deserialization
│ ├── error.rs # Error types
│ └── operations.rs # Tree operations implementation
├── tests/
│ ├── basic_test.rs # Basic operations tests
│ ├── prefix_test.rs # Prefix operations tests
│ └── edge_cases.rs # Edge case tests
└── examples/
├── basic.rs # Basic usage example
├── prefix.rs # Prefix operations example
└── performance.rs # Performance benchmark
```
## 8. Dependencies
The Rust implementation will use the following dependencies:
- `ourdb` for persistent storage
- `thiserror` for error handling
- `log` for logging
- `criterion` for benchmarking (dev dependency)
## 9. Compatibility Considerations
To ensure compatibility with the V implementation:
1. Maintain the same serialization format for nodes
2. Ensure identical behavior for all operations
3. Support reading existing RadixTree data
4. Maintain the same performance characteristics
## 10. Future Extensions
Potential future extensions to consider:
1. Async API for non-blocking operations
2. Iterator interface for efficient traversal
3. Batch operations for improved performance
4. Custom serialization formats for specific use cases
5. Compression support for values
6. Concurrency support for parallel operations
## 11. Conclusion
This architecture provides a roadmap for porting RadixTree from V to Rust while maintaining compatibility and leveraging Rust's strengths. The implementation will follow a phased approach, starting with core data structures and gradually building up to the complete system.
The Rust implementation aims to be:
- **Safe**: Leveraging Rust's ownership model for memory safety
- **Fast**: Maintaining or improving performance compared to V
- **Compatible**: Working with existing RadixTree data
- **Extensible**: Providing a foundation for future enhancements
- **Well-tested**: Including comprehensive test coverage
## 12. Implementation Files
### 12.1 Cargo.toml
```toml
[package]
name = "radixtree"
version = "0.1.0"
edition = "2021"
description = "A persistent radix tree implementation using OurDB for storage"
authors = ["OurWorld Team"]
[dependencies]
ourdb = { path = "../ourdb" }
thiserror = "1.0.40"
log = "0.4.17"
[dev-dependencies]
criterion = "0.5.1"
[[bench]]
name = "radixtree_benchmarks"
harness = false
[[example]]
name = "basic_usage"
path = "examples/basic_usage.rs"
[[example]]
name = "prefix_operations"
path = "examples/prefix_operations.rs"
```
### 12.2 src/lib.rs
```rust
//! RadixTree is a space-optimized tree data structure that enables efficient string key operations
//! with persistent storage using OurDB as a backend.
//!
//! This implementation provides a persistent radix tree that can be used for efficient
//! prefix-based key operations, such as auto-complete, routing tables, and more.
mod error;
mod node;
mod operations;
mod serialize;
pub use error::Error;
pub use node::{Node, NodeRef};
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use std::path::PathBuf;
/// RadixTree represents a radix tree data structure with persistent storage.
pub struct RadixTree {
db: OurDB,
root_id: u32,
}
impl RadixTree {
/// Creates a new radix tree with the specified database path.
///
/// # Arguments
///
/// * `path` - The path to the database directory
/// * `reset` - Whether to reset the database if it exists
///
/// # Returns
///
/// A new `RadixTree` instance
///
/// # Errors
///
/// Returns an error if the database cannot be created or opened
pub fn new(path: &str, reset: bool) -> Result<Self, Error> {
// Implementation will go here
unimplemented!()
}
/// Sets a key-value pair in the tree.
///
/// # Arguments
///
/// * `key` - The key to set
/// * `value` - The value to set
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn set(&mut self, key: &str, value: Vec<u8>) -> Result<(), Error> {
// Implementation will go here
unimplemented!()
}
/// Gets a value by key from the tree.
///
/// # Arguments
///
/// * `key` - The key to get
///
/// # Returns
///
/// The value associated with the key
///
/// # Errors
///
/// Returns an error if the key is not found or the operation fails
pub fn get(&mut self, key: &str) -> Result<Vec<u8>, Error> {
// Implementation will go here
unimplemented!()
}
/// Updates the value at a given key prefix.
///
/// # Arguments
///
/// * `prefix` - The key prefix to update
/// * `new_value` - The new value to set
///
/// # Errors
///
/// Returns an error if the prefix is not found or the operation fails
pub fn update(&mut self, prefix: &str, new_value: Vec<u8>) -> Result<(), Error> {
// Implementation will go here
unimplemented!()
}
/// Deletes a key from the tree.
///
/// # Arguments
///
/// * `key` - The key to delete
///
/// # Errors
///
/// Returns an error if the key is not found or the operation fails
pub fn delete(&mut self, key: &str) -> Result<(), Error> {
// Implementation will go here
unimplemented!()
}
/// Lists all keys with a given prefix.
///
/// # Arguments
///
/// * `prefix` - The prefix to search for
///
/// # Returns
///
/// A list of keys that start with the given prefix
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn list(&mut self, prefix: &str) -> Result<Vec<String>, Error> {
// Implementation will go here
unimplemented!()
}
/// Gets all values for keys with a given prefix.
///
/// # Arguments
///
/// * `prefix` - The prefix to search for
///
/// # Returns
///
/// A list of values for keys that start with the given prefix
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn getall(&mut self, prefix: &str) -> Result<Vec<Vec<u8>>, Error> {
// Implementation will go here
unimplemented!()
}
}
```
### 12.3 src/error.rs
```rust
//! Error types for the RadixTree module.
use thiserror::Error;
/// Error type for RadixTree operations.
#[derive(Debug, Error)]
pub enum Error {
/// Error from OurDB operations.
#[error("OurDB error: {0}")]
OurDB(#[from] ourdb::Error),
/// Error when a key is not found.
#[error("Key not found: {0}")]
KeyNotFound(String),
/// Error when a prefix is not found.
#[error("Prefix not found: {0}")]
PrefixNotFound(String),
/// Error during serialization.
#[error("Serialization error: {0}")]
Serialization(String),
/// Error during deserialization.
#[error("Deserialization error: {0}")]
Deserialization(String),
/// Error for invalid operations.
#[error("Invalid operation: {0}")]
InvalidOperation(String),
}
```
### 12.4 src/node.rs
```rust
//! Node types for the RadixTree module.
/// Represents a node in the radix tree.
pub struct Node {
/// The segment of the key stored at this node.
pub key_segment: String,
/// Value stored at this node (empty if not a leaf).
pub value: Vec<u8>,
/// References to child nodes.
pub children: Vec<NodeRef>,
/// Whether this node is a leaf node.
pub is_leaf: bool,
}
/// Reference to a node in the database.
pub struct NodeRef {
/// The key segment for this child.
pub key_part: String,
/// Database ID of the node.
pub node_id: u32,
}
impl Node {
/// Creates a new node.
pub fn new(key_segment: String, value: Vec<u8>, is_leaf: bool) -> Self {
Self {
key_segment,
value,
children: Vec::new(),
is_leaf,
}
}
/// Creates a new root node.
pub fn new_root() -> Self {
Self {
key_segment: String::new(),
value: Vec::new(),
children: Vec::new(),
is_leaf: false,
}
}
}
impl NodeRef {
/// Creates a new node reference.
pub fn new(key_part: String, node_id: u32) -> Self {
Self {
key_part,
node_id,
}
}
}
```
### 12.5 src/serialize.rs
```rust
//! Serialization and deserialization for RadixTree nodes.
use crate::error::Error;
use crate::node::{Node, NodeRef};
/// Current binary format version.
const VERSION: u8 = 1;
impl Node {
/// Serializes a node to bytes for storage.
pub fn serialize(&self) -> Vec<u8> {
// Implementation will go here
unimplemented!()
}
/// Deserializes bytes to a node.
pub fn deserialize(data: &[u8]) -> Result<Self, Error> {
// Implementation will go here
unimplemented!()
}
}
```
### 12.6 src/operations.rs
```rust
//! Implementation of RadixTree operations.
use crate::error::Error;
use crate::node::{Node, NodeRef};
use crate::RadixTree;
impl RadixTree {
/// Helper function to get a node from the database.
pub(crate) fn get_node(&mut self, node_id: u32) -> Result<Node, Error> {
// Implementation will go here
unimplemented!()
}
/// Helper function to save a node to the database.
pub(crate) fn save_node(&mut self, node_id: Option<u32>, node: &Node) -> Result<u32, Error> {
// Implementation will go here
unimplemented!()
}
/// Helper function to find all keys with a given prefix.
fn find_keys_with_prefix(
&mut self,
node_id: u32,
current_path: &str,
prefix: &str,
result: &mut Vec<String>,
) -> Result<(), Error> {
// Implementation will go here
unimplemented!()
}
/// Helper function to recursively collect all keys under a node.
fn collect_all_keys(
&mut self,
node_id: u32,
current_path: &str,
result: &mut Vec<String>,
) -> Result<(), Error> {
// Implementation will go here
unimplemented!()
}
/// Helper function to get the common prefix of two strings.
fn get_common_prefix(a: &str, b: &str) -> String {
// Implementation will go here
unimplemented!()
}
}
```
### 12.7 examples/basic_usage.rs
```rust
//! Basic usage example for RadixTree.
use radixtree::RadixTree;
fn main() -> Result<(), radixtree::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("radixtree_example");
std::fs::create_dir_all(&db_path)?;
println!("Creating radix tree at: {}", db_path.display());
// Create a new radix tree
let mut tree = RadixTree::new(db_path.to_str().unwrap(), true)?;
// Store some data
tree.set("hello", b"world".to_vec())?;
tree.set("help", b"me".to_vec())?;
tree.set("helicopter", b"flying".to_vec())?;
// Retrieve and print the data
let value = tree.get("hello")?;
println!("hello: {}", String::from_utf8_lossy(&value));
// List keys with prefix
let keys = tree.list("hel")?;
println!("Keys with prefix 'hel': {:?}", keys);
// Get all values with prefix
let values = tree.getall("hel")?;
println!("Values with prefix 'hel':");
for (i, value) in values.iter().enumerate() {
println!(" {}: {}", i, String::from_utf8_lossy(value));
}
// Delete a key
tree.delete("help")?;
println!("Deleted 'help'");
// Verify deletion
let keys_after = tree.list("hel")?;
println!("Keys with prefix 'hel' after deletion: {:?}", keys_after);
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("Cleaned up database directory");
} else {
println!("Database kept at: {}", db_path.display());
}
Ok(())
}
```

815
radixtree/Cargo.lock generated
View File

@@ -1,815 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "aho-corasick"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
dependencies = [
"memchr",
]
[[package]]
name = "anes"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstyle"
version = "1.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9"
[[package]]
name = "autocfg"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "bitflags"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd"
[[package]]
name = "bumpalo"
version = "3.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf"
[[package]]
name = "cast"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "ciborium"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e"
dependencies = [
"ciborium-io",
"ciborium-ll",
"serde",
]
[[package]]
name = "ciborium-io"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757"
[[package]]
name = "ciborium-ll"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9"
dependencies = [
"ciborium-io",
"half",
]
[[package]]
name = "clap"
version = "4.5.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944"
dependencies = [
"clap_builder",
]
[[package]]
name = "clap_builder"
version = "4.5.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9"
dependencies = [
"anstyle",
"clap_lex",
]
[[package]]
name = "clap_lex"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "crc32fast"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]]
name = "criterion"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f"
dependencies = [
"anes",
"cast",
"ciborium",
"clap",
"criterion-plot",
"is-terminal",
"itertools",
"num-traits",
"once_cell",
"oorandom",
"plotters",
"rayon",
"regex",
"serde",
"serde_derive",
"serde_json",
"tinytemplate",
"walkdir",
]
[[package]]
name = "criterion-plot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
dependencies = [
"cast",
"itertools",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
[[package]]
name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
[[package]]
name = "errno"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e"
dependencies = [
"libc",
"windows-sys",
]
[[package]]
name = "fastrand"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "getrandom"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0"
dependencies = [
"cfg-if",
"libc",
"r-efi",
"wasi 0.14.2+wasi-0.2.4",
]
[[package]]
name = "half"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9"
dependencies = [
"cfg-if",
"crunchy",
]
[[package]]
name = "hermit-abi"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e"
[[package]]
name = "is-terminal"
version = "0.4.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi",
"libc",
"windows-sys",
]
[[package]]
name = "itertools"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "js-sys"
version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f"
dependencies = [
"once_cell",
"wasm-bindgen",
]
[[package]]
name = "libc"
version = "0.2.171"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
[[package]]
name = "linux-raw-sys"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413"
[[package]]
name = "log"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "memchr"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
[[package]]
name = "oorandom"
version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "ourdb"
version = "0.1.0"
dependencies = [
"crc32fast",
"log",
"rand",
"thiserror",
]
[[package]]
name = "plotters"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747"
dependencies = [
"num-traits",
"plotters-backend",
"plotters-svg",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "plotters-backend"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a"
[[package]]
name = "plotters-svg"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670"
dependencies = [
"plotters-backend",
]
[[package]]
name = "ppv-lite86"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy",
]
[[package]]
name = "proc-macro2"
version = "1.0.94"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
dependencies = [
"proc-macro2",
]
[[package]]
name = "r-efi"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5"
[[package]]
name = "radixtree"
version = "0.1.0"
dependencies = [
"criterion",
"log",
"ourdb",
"tempfile",
"thiserror",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom 0.2.15",
]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]]
name = "regex"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata",
"regex-syntax",
]
[[package]]
name = "regex-automata"
version = "0.4.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
[[package]]
name = "regex-syntax"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "rustix"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "rustversion"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2"
[[package]]
name = "ryu"
version = "1.0.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "serde"
version = "1.0.219"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.219"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.140"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
dependencies = [
"itoa",
"memchr",
"ryu",
"serde",
]
[[package]]
name = "syn"
version = "2.0.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "tempfile"
version = "3.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf"
dependencies = [
"fastrand",
"getrandom 0.3.2",
"once_cell",
"rustix",
"windows-sys",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tinytemplate"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "unicode-ident"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
[[package]]
name = "walkdir"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasi"
version = "0.14.2+wasi-0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3"
dependencies = [
"wit-bindgen-rt",
]
[[package]]
name = "wasm-bindgen"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5"
dependencies = [
"cfg-if",
"once_cell",
"rustversion",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6"
dependencies = [
"bumpalo",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d"
dependencies = [
"unicode-ident",
]
[[package]]
name = "web-sys"
version = "0.3.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "winapi-util"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys",
]
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "wit-bindgen-rt"
version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1"
dependencies = [
"bitflags",
]
[[package]]
name = "zerocopy"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
dependencies = [
"proc-macro2",
"quote",
"syn",
]

27
radixtree/Cargo.toml
View File

@@ -1,27 +0,0 @@
[package]
name = "radixtree"
version = "0.1.0"
edition = "2021"
description = "A persistent radix tree implementation using OurDB for storage"
authors = ["OurWorld Team"]
[dependencies]
ourdb = { path = "../ourdb" }
thiserror = "1.0.40"
log = "0.4.17"
[dev-dependencies]
criterion = "0.5.1"
tempfile = "3.8.0"
[[bench]]
name = "radixtree_benchmarks"
harness = false
[[example]]
name = "basic_usage"
path = "examples/basic_usage.rs"
[[example]]
name = "prefix_operations"
path = "examples/prefix_operations.rs"

265
radixtree/MIGRATION.md
View File

@@ -1,265 +0,0 @@
# Migration Guide: V to Rust RadixTree
This document provides guidance for migrating from the V implementation of RadixTree to the Rust implementation.
## API Changes
The Rust implementation maintains API compatibility with the V implementation, but with some idiomatic Rust changes:
### V API
```v
// Create a new radix tree
mut rt := radixtree.new(path: '/tmp/radixtree_test', reset: true)!
// Set a key-value pair
rt.set('test', 'value1'.bytes())!
// Get a value by key
value := rt.get('test')!
// Update a value at a prefix
rt.update('prefix', 'new_value'.bytes())!
// Delete a key
rt.delete('test')!
// List keys with a prefix
keys := rt.list('prefix')!
// Get all values with a prefix
values := rt.getall('prefix')!
```
### Rust API
```rust
// Create a new radix tree
let mut tree = RadixTree::new("/tmp/radixtree_test", true)?;
// Set a key-value pair
tree.set("test", b"value1".to_vec())?;
// Get a value by key
let value = tree.get("test")?;
// Update a value at a prefix
tree.update("prefix", b"new_value".to_vec())?;
// Delete a key
tree.delete("test")?;
// List keys with a prefix
let keys = tree.list("prefix")?;
// Get all values with a prefix
let values = tree.getall("prefix")?;
```
## Key Differences
1. **Error Handling**: The Rust implementation uses Rust's `Result` type for error handling, while the V implementation uses V's `!` operator.
2. **String Handling**: The Rust implementation uses Rust's `&str` for string parameters and `String` for string return values, while the V implementation uses V's `string` type.
3. **Binary Data**: The Rust implementation uses Rust's `Vec<u8>` for binary data, while the V implementation uses V's `[]u8` type.
4. **Constructor**: The Rust implementation uses a constructor function with separate parameters, while the V implementation uses a struct with named parameters.
5. **Ownership**: The Rust implementation follows Rust's ownership model, requiring mutable references for methods that modify the tree.
## Data Compatibility
The Rust implementation maintains data compatibility with the V implementation:
- The same serialization format is used for nodes
- The same OurDB storage format is used
- Existing RadixTree data created with the V implementation can be read by the Rust implementation
## Migration Steps
1. **Update Dependencies**: Replace the V RadixTree dependency with the Rust RadixTree dependency in your project.
2. **Update Import Statements**: Replace V import statements with Rust use statements.
```v
// V
import freeflowuniverse.herolib.data.radixtree
```
```rust
// Rust
use radixtree::RadixTree;
```
3. **Update Constructor Calls**: Replace V constructor calls with Rust constructor calls.
```v
// V
mut rt := radixtree.new(path: '/path/to/db', reset: false)!
```
```rust
// Rust
let mut tree = RadixTree::new("/path/to/db", false)?;
```
4. **Update Method Calls**: Replace V method calls with Rust method calls.
```v
// V
rt.set('key', 'value'.bytes())!
```
```rust
// Rust
tree.set("key", b"value".to_vec())?;
```
5. **Update Error Handling**: Replace V error handling with Rust error handling.
```v
// V
if value := rt.get('key') {
println('Found: ${value.bytestr()}')
} else {
println('Error: ${err}')
}
```
```rust
// Rust
match tree.get("key") {
Ok(value) => println!("Found: {}", String::from_utf8_lossy(&value)),
Err(e) => println!("Error: {}", e),
}
```
6. **Update String Conversions**: Replace V string conversions with Rust string conversions.
```v
// V
value.bytestr() // Convert []u8 to string
```
```rust
// Rust
String::from_utf8_lossy(&value) // Convert Vec<u8> to string
```
## Example Migration
### V Code
```v
module main
import freeflowuniverse.herolib.data.radixtree
fn main() {
mut rt := radixtree.new(path: '/tmp/radixtree_test', reset: true) or {
println('Error creating RadixTree: ${err}')
return
}
rt.set('hello', 'world'.bytes()) or {
println('Error setting key: ${err}')
return
}
rt.set('help', 'me'.bytes()) or {
println('Error setting key: ${err}')
return
}
if value := rt.get('hello') {
println('hello: ${value.bytestr()}')
} else {
println('Error getting key: ${err}')
return
}
keys := rt.list('hel') or {
println('Error listing keys: ${err}')
return
}
println('Keys with prefix "hel": ${keys}')
values := rt.getall('hel') or {
println('Error getting all values: ${err}')
return
}
println('Values with prefix "hel":')
for i, value in values {
println(' ${i}: ${value.bytestr()}')
}
rt.delete('help') or {
println('Error deleting key: ${err}')
return
}
println('Deleted "help"')
}
```
### Rust Code
```rust
use radixtree::RadixTree;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut tree = RadixTree::new("/tmp/radixtree_test", true)
.map_err(|e| format!("Error creating RadixTree: {}", e))?;
tree.set("hello", b"world".to_vec())
.map_err(|e| format!("Error setting key: {}", e))?;
tree.set("help", b"me".to_vec())
.map_err(|e| format!("Error setting key: {}", e))?;
let value = tree.get("hello")
.map_err(|e| format!("Error getting key: {}", e))?;
println!("hello: {}", String::from_utf8_lossy(&value));
let keys = tree.list("hel")
.map_err(|e| format!("Error listing keys: {}", e))?;
println!("Keys with prefix \"hel\": {:?}", keys);
let values = tree.getall("hel")
.map_err(|e| format!("Error getting all values: {}", e))?;
println!("Values with prefix \"hel\":");
for (i, value) in values.iter().enumerate() {
println!(" {}: {}", i, String::from_utf8_lossy(value));
}
tree.delete("help")
.map_err(|e| format!("Error deleting key: {}", e))?;
println!("Deleted \"help\"");
Ok(())
}
```
## Performance Considerations
The Rust implementation should provide similar or better performance compared to the V implementation. However, there are some considerations:
1. **Memory Usage**: The Rust implementation may have different memory usage patterns due to Rust's ownership model.
2. **Error Handling**: The Rust implementation uses Rust's `Result` type, which may have different performance characteristics compared to V's error handling.
3. **String Handling**: The Rust implementation uses Rust's string types, which may have different performance characteristics compared to V's string types.
## Troubleshooting
If you encounter issues during migration, check the following:
1. **Data Compatibility**: Ensure that the data format is compatible between the V and Rust implementations.
2. **API Usage**: Ensure that you're using the correct API for the Rust implementation.
3. **Error Handling**: Ensure that you're handling errors correctly in the Rust implementation.
4. **String Encoding**: Ensure that string encoding is consistent between the V and Rust implementations.
If you encounter any issues that are not covered in this guide, please report them to the project maintainers.

189
radixtree/README.md
View File

@@ -1,189 +0,0 @@
# RadixTree
A persistent radix tree implementation in Rust using OurDB for storage.
## Overview
RadixTree is a space-optimized tree data structure that enables efficient string key operations with persistent storage. This implementation provides a persistent radix tree that can be used for efficient prefix-based key operations, such as auto-complete, routing tables, and more.
A radix tree (also known as a patricia trie or radix trie) is a space-optimized tree data structure that enables efficient string key operations. Unlike a standard trie where each node represents a single character, a radix tree compresses paths by allowing nodes to represent multiple characters (key segments).
Key characteristics:
- Each node stores a segment of a key (not just a single character)
- Nodes can have multiple children, each representing a different branch
- Leaf nodes contain the actual values
- Optimizes storage by compressing common prefixes
## Features
- Efficient prefix-based key operations
- Persistent storage using OurDB backend
- Memory-efficient storage of strings with common prefixes
- Support for binary values
- Thread-safe operations through OurDB
## Usage
Add the dependency to your `Cargo.toml`:
```toml
[dependencies]
radixtree = { path = "../radixtree" }
```
### Basic Example
```rust
use radixtree::RadixTree;
fn main() -> Result<(), radixtree::Error> {
// Create a new radix tree
let mut tree = RadixTree::new("/tmp/radix", false)?;
// Set key-value pairs
tree.set("hello", b"world".to_vec())?;
tree.set("help", b"me".to_vec())?;
// Get values by key
let value = tree.get("hello")?;
println!("hello: {}", String::from_utf8_lossy(&value)); // Prints: world
// List keys by prefix
let keys = tree.list("hel")?; // Returns ["hello", "help"]
println!("Keys with prefix 'hel': {:?}", keys);
// Get all values by prefix
let values = tree.getall("hel")?; // Returns [b"world", b"me"]
// Delete keys
tree.delete("help")?;
Ok(())
}
```
## API
### Creating a RadixTree
```rust
// Create a new radix tree
let mut tree = RadixTree::new("/tmp/radix", false)?;
// Create a new radix tree and reset if it exists
let mut tree = RadixTree::new("/tmp/radix", true)?;
```
### Setting Values
```rust
// Set a key-value pair
tree.set("key", b"value".to_vec())?;
```
### Getting Values
```rust
// Get a value by key
let value = tree.get("key")?;
```
### Updating Values
```rust
// Update a value at a given prefix
tree.update("prefix", b"new_value".to_vec())?;
```
### Deleting Keys
```rust
// Delete a key
tree.delete("key")?;
```
### Listing Keys by Prefix
```rust
// List all keys with a given prefix
let keys = tree.list("prefix")?;
```
### Getting All Values by Prefix
```rust
// Get all values for keys with a given prefix
let values = tree.getall("prefix")?;
```
## Performance Characteristics
- Search: O(k) where k is the key length
- Insert: O(k) for new keys, may require node splitting
- Delete: O(k) plus potential node cleanup
- Space: O(n) where n is the total length of all keys
## Use Cases
RadixTree is particularly useful for:
- Prefix-based searching
- IP routing tables
- Dictionary implementations
- Auto-complete systems
- File system paths
- Any application requiring efficient string key operations with persistence
## Implementation Details
The RadixTree implementation uses OurDB for persistent storage:
- Each node is serialized and stored as a record in OurDB
- Node references use OurDB record IDs
- The tree maintains a root node ID for traversal
- Node serialization includes version tracking for format evolution
For more detailed information about the implementation, see the [ARCHITECTURE.md](./ARCHITECTURE.md) file.
## Running Tests
The project includes a comprehensive test suite that verifies all functionality:
```bash
# Run all tests
cargo test
# Run specific test file
cargo test --test basic_test
cargo test --test prefix_test
cargo test --test getall_test
cargo test --test serialize_test
```
## Running Examples
The project includes example applications that demonstrate how to use the RadixTree:
```bash
# Run the basic usage example
cargo run --example basic_usage
# Run the prefix operations example
cargo run --example prefix_operations
```
## Benchmarking
The project includes benchmarks to measure performance:
```bash
# Run all benchmarks
cargo bench
# Run specific benchmark
cargo bench -- set
cargo bench -- get
cargo bench -- prefix_operations
```
## License
This project is licensed under the same license as the HeroCode project.

141
radixtree/benches/radixtree_benchmarks.rs
View File

@@ -1,141 +0,0 @@
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use radixtree::RadixTree;
use std::path::PathBuf;
use tempfile::tempdir;
fn criterion_benchmark(c: &mut Criterion) {
// Create a temporary directory for benchmarks
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Benchmark set operation
c.bench_function("set", |b| {
let mut tree = RadixTree::new(db_path, true).unwrap();
let mut i = 0;
b.iter(|| {
let key = format!("benchmark_key_{}", i);
let value = format!("benchmark_value_{}", i).into_bytes();
tree.set(&key, value).unwrap();
i += 1;
});
});
// Setup tree with data for get/list/delete benchmarks
let mut setup_tree = RadixTree::new(db_path, true).unwrap();
for i in 0..1000 {
let key = format!("benchmark_key_{}", i);
let value = format!("benchmark_value_{}", i).into_bytes();
setup_tree.set(&key, value).unwrap();
}
// Benchmark get operation
c.bench_function("get", |b| {
let mut tree = RadixTree::new(db_path, false).unwrap();
let mut i = 0;
b.iter(|| {
let key = format!("benchmark_key_{}", i % 1000);
let _value = tree.get(&key).unwrap();
i += 1;
});
});
// Benchmark list operation
c.bench_function("list", |b| {
let mut tree = RadixTree::new(db_path, false).unwrap();
b.iter(|| {
let _keys = tree.list("benchmark_key_1").unwrap();
});
});
// Benchmark getall operation
c.bench_function("getall", |b| {
let mut tree = RadixTree::new(db_path, false).unwrap();
b.iter(|| {
let _values = tree.getall("benchmark_key_1").unwrap();
});
});
// Benchmark update operation
c.bench_function("update", |b| {
let mut tree = RadixTree::new(db_path, false).unwrap();
let mut i = 0;
b.iter(|| {
let key = format!("benchmark_key_{}", i % 1000);
let new_value = format!("updated_value_{}", i).into_bytes();
tree.update(&key, new_value).unwrap();
i += 1;
});
});
// Benchmark delete operation
c.bench_function("delete", |b| {
// Create a fresh tree for deletion benchmarks
let delete_dir = tempdir().expect("Failed to create temp directory");
let delete_path = delete_dir.path().to_str().unwrap();
let mut tree = RadixTree::new(delete_path, true).unwrap();
// Setup keys to delete
for i in 0..1000 {
let key = format!("delete_key_{}", i);
let value = format!("delete_value_{}", i).into_bytes();
tree.set(&key, value).unwrap();
}
let mut i = 0;
b.iter(|| {
let key = format!("delete_key_{}", i % 1000);
// Only try to delete if it exists
if tree.get(&key).is_ok() {
tree.delete(&key).unwrap();
}
i += 1;
});
});
// Benchmark prefix operations with varying tree sizes
let mut group = c.benchmark_group("prefix_operations");
for &size in &[100, 1000, 10000] {
// Create a fresh tree for each size
let size_dir = tempdir().expect("Failed to create temp directory");
let size_path = size_dir.path().to_str().unwrap();
let mut tree = RadixTree::new(size_path, true).unwrap();
// Insert data with common prefixes
for i in 0..size {
let prefix = match i % 5 {
0 => "user",
1 => "post",
2 => "comment",
3 => "product",
_ => "category",
};
let key = format!("{}_{}", prefix, i);
let value = format!("value_{}", i).into_bytes();
tree.set(&key, value).unwrap();
}
// Benchmark list operation for this size
group.bench_function(format!("list_size_{}", size), |b| {
b.iter(|| {
for prefix in &["user", "post", "comment", "product", "category"] {
let _keys = tree.list(prefix).unwrap();
}
});
});
// Benchmark getall operation for this size
group.bench_function(format!("getall_size_{}", size), |b| {
b.iter(|| {
for prefix in &["user", "post", "comment", "product", "category"] {
let _values = tree.getall(prefix).unwrap();
}
});
});
}
group.finish();
}
criterion_group!(benches, criterion_benchmark);
criterion_main!(benches);

51
radixtree/examples/basic_usage.rs
View File

@@ -1,51 +0,0 @@
use radixtree::RadixTree;
use std::path::PathBuf;
fn main() -> Result<(), radixtree::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("radixtree_example");
std::fs::create_dir_all(&db_path)?;
println!("Creating radix tree at: {}", db_path.display());
// Create a new radix tree
let mut tree = RadixTree::new(db_path.to_str().unwrap(), true)?;
// Store some data
println!("Storing data...");
tree.set("hello", b"world".to_vec())?;
tree.set("help", b"me".to_vec())?;
tree.set("helicopter", b"flying".to_vec())?;
// Retrieve and print the data
let value = tree.get("hello")?;
println!("hello: {}", String::from_utf8_lossy(&value));
// Update a value
println!("Updating value...");
tree.update("hello", b"updated world".to_vec())?;
// Retrieve the updated value
let updated_value = tree.get("hello")?;
println!("hello (updated): {}", String::from_utf8_lossy(&updated_value));
// Delete a key
println!("Deleting 'help'...");
tree.delete("help")?;
// Try to retrieve the deleted key (should fail)
match tree.get("help") {
Ok(value) => println!("Unexpected: help still exists with value: {}", String::from_utf8_lossy(&value)),
Err(e) => println!("As expected, help was deleted: {}", e),
}
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("Cleaned up database directory");
} else {
println!("Database kept at: {}", db_path.display());
}
Ok(())
}

121
radixtree/examples/large_scale_test.rs
View File

@@ -1,121 +0,0 @@
use radixtree::RadixTree;
use std::time::{Duration, Instant};
use std::io::{self, Write};
// Use much smaller batches to avoid hitting OurDB's size limit
const BATCH_SIZE: usize = 1_000;
const NUM_BATCHES: usize = 1_000; // Total records: 1,000,000
const PROGRESS_INTERVAL: usize = 100;
fn main() -> Result<(), radixtree::Error> {
// Overall metrics
let total_start_time = Instant::now();
let mut total_records_inserted = 0;
let mut batch_times = Vec::with_capacity(NUM_BATCHES);
println!("Will insert up to {} records in batches of {}",
BATCH_SIZE * NUM_BATCHES, BATCH_SIZE);
// Process in batches to avoid OurDB size limits
for batch in 0..NUM_BATCHES {
// Create a new database for each batch
let batch_path = std::env::temp_dir().join(format!("radixtree_batch_{}", batch));
// Clean up any existing database
if batch_path.exists() {
std::fs::remove_dir_all(&batch_path)?;
}
std::fs::create_dir_all(&batch_path)?;
println!("\nBatch {}/{}: Creating new radix tree...", batch + 1, NUM_BATCHES);
let mut tree = RadixTree::new(batch_path.to_str().unwrap(), true)?;
let batch_start_time = Instant::now();
let mut last_progress_time = Instant::now();
let mut last_progress_count = 0;
// Insert records for this batch
for i in 0..BATCH_SIZE {
let global_index = batch * BATCH_SIZE + i;
let key = format!("key:{:08}", global_index);
let value = format!("val{}", global_index).into_bytes();
tree.set(&key, value)?;
// Show progress at intervals
if (i + 1) % PROGRESS_INTERVAL == 0 || i == BATCH_SIZE - 1 {
let records_since_last = i + 1 - last_progress_count;
let time_since_last = last_progress_time.elapsed();
let records_per_second = records_since_last as f64 / time_since_last.as_secs_f64();
print!("\rProgress: {}/{} records ({:.2}%) - {:.2} records/sec",
i + 1, BATCH_SIZE,
(i + 1) as f64 / BATCH_SIZE as f64 * 100.0,
records_per_second);
io::stdout().flush().unwrap();
last_progress_time = Instant::now();
last_progress_count = i + 1;
}
}
let batch_duration = batch_start_time.elapsed();
batch_times.push(batch_duration);
total_records_inserted += BATCH_SIZE;
println!("\nBatch {}/{} completed in {:?} ({:.2} records/sec)",
batch + 1, NUM_BATCHES,
batch_duration,
BATCH_SIZE as f64 / batch_duration.as_secs_f64());
// Test random access performance for this batch
println!("Testing access performance for batch {}...", batch + 1);
let mut total_get_time = Duration::new(0, 0);
let num_samples = 100;
// Use a simple distribution pattern
for i in 0..num_samples {
// Distribute samples across the batch
let sample_id = batch * BATCH_SIZE + (i * (BATCH_SIZE / num_samples));
let key = format!("key:{:08}", sample_id);
let get_start = Instant::now();
let _ = tree.get(&key)?;
total_get_time += get_start.elapsed();
}
println!("Average time to retrieve a record: {:?}",
total_get_time / num_samples as u32);
// Test prefix search performance
println!("Testing prefix search performance...");
let prefix = format!("key:{:02}", batch % 100);
let list_start = Instant::now();
let keys = tree.list(&prefix)?;
let list_duration = list_start.elapsed();
println!("Found {} keys with prefix '{}' in {:?}",
keys.len(), prefix, list_duration);
}
// Overall performance summary
let total_duration = total_start_time.elapsed();
println!("\n\nPerformance Summary:");
println!("Total time to insert {} records: {:?}", total_records_inserted, total_duration);
println!("Average insertion rate: {:.2} records/second",
total_records_inserted as f64 / total_duration.as_secs_f64());
// Show performance trend
println!("\nPerformance Trend (batch number vs. time):");
for (i, duration) in batch_times.iter().enumerate() {
if i % 10 == 0 || i == batch_times.len() - 1 { // Only show every 10th point
println!(" Batch {}: {:?} ({:.2} records/sec)",
i + 1,
duration,
BATCH_SIZE as f64 / duration.as_secs_f64());
}
}
Ok(())
}

134
radixtree/examples/performance_test.rs
View File

@@ -1,134 +0,0 @@
use radixtree::RadixTree;
use std::time::{Duration, Instant};
use std::io::{self, Write};
// Number of records to insert
const TOTAL_RECORDS: usize = 1_000_000;
// How often to report progress (every X records)
const PROGRESS_INTERVAL: usize = 10_000;
// How many records to use for performance sampling
const PERFORMANCE_SAMPLE_SIZE: usize = 1000;
fn main() -> Result<(), radixtree::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("radixtree_performance_test");
// Completely remove and recreate the directory to ensure a clean start
if db_path.exists() {
std::fs::remove_dir_all(&db_path)?;
}
std::fs::create_dir_all(&db_path)?;
println!("Creating radix tree at: {}", db_path.display());
println!("Will insert {} records and show progress...", TOTAL_RECORDS);
// Create a new radix tree
let mut tree = RadixTree::new(db_path.to_str().unwrap(), true)?;
// Track overall time
let start_time = Instant::now();
// Track performance metrics
let mut insertion_times = Vec::with_capacity(TOTAL_RECORDS / PROGRESS_INTERVAL);
let mut last_batch_time = Instant::now();
let mut last_batch_records = 0;
// Insert records and track progress
for i in 0..TOTAL_RECORDS {
let key = format!("key:{:08}", i);
// Use smaller values to avoid exceeding OurDB's size limit
let value = format!("val{}", i).into_bytes();
// Time the insertion of every Nth record for performance sampling
if i % PERFORMANCE_SAMPLE_SIZE == 0 {
let insert_start = Instant::now();
tree.set(&key, value)?;
let insert_duration = insert_start.elapsed();
// Only print detailed timing for specific samples to avoid flooding output
if i % (PERFORMANCE_SAMPLE_SIZE * 10) == 0 {
println!("Record {}: Insertion took {:?}", i, insert_duration);
}
} else {
tree.set(&key, value)?;
}
// Show progress at intervals
if (i + 1) % PROGRESS_INTERVAL == 0 || i == TOTAL_RECORDS - 1 {
let records_in_batch = i + 1 - last_batch_records;
let batch_duration = last_batch_time.elapsed();
let records_per_second = records_in_batch as f64 / batch_duration.as_secs_f64();
insertion_times.push((i + 1, batch_duration));
print!("\rProgress: {}/{} records ({:.2}%) - {:.2} records/sec",
i + 1, TOTAL_RECORDS,
(i + 1) as f64 / TOTAL_RECORDS as f64 * 100.0,
records_per_second);
io::stdout().flush().unwrap();
last_batch_time = Instant::now();
last_batch_records = i + 1;
}
}
let total_duration = start_time.elapsed();
println!("\n\nPerformance Summary:");
println!("Total time to insert {} records: {:?}", TOTAL_RECORDS, total_duration);
println!("Average insertion rate: {:.2} records/second",
TOTAL_RECORDS as f64 / total_duration.as_secs_f64());
// Show performance trend
println!("\nPerformance Trend (records inserted vs. time per batch):");
for (i, (record_count, duration)) in insertion_times.iter().enumerate() {
if i % 10 == 0 || i == insertion_times.len() - 1 { // Only show every 10th point to avoid too much output
println!(" After {} records: {:?} for {} records ({:.2} records/sec)",
record_count,
duration,
PROGRESS_INTERVAL,
PROGRESS_INTERVAL as f64 / duration.as_secs_f64());
}
}
// Test access performance with distributed samples
println!("\nTesting access performance with distributed samples...");
let mut total_get_time = Duration::new(0, 0);
let num_samples = 1000;
// Use a simple distribution pattern instead of random
for i in 0..num_samples {
// Distribute samples across the entire range
let sample_id = (i * (TOTAL_RECORDS / num_samples)) % TOTAL_RECORDS;
let key = format!("key:{:08}", sample_id);
let get_start = Instant::now();
let _ = tree.get(&key)?;
total_get_time += get_start.elapsed();
}
println!("Average time to retrieve a record: {:?}",
total_get_time / num_samples as u32);
// Test prefix search performance
println!("\nTesting prefix search performance...");
let prefixes = ["key:0", "key:1", "key:5", "key:9"];
for prefix in &prefixes {
let list_start = Instant::now();
let keys = tree.list(prefix)?;
let list_duration = list_start.elapsed();
println!("Found {} keys with prefix '{}' in {:?}",
keys.len(), prefix, list_duration);
}
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("\nCleaned up database directory");
} else {
println!("\nDatabase kept at: {}", db_path.display());
}
Ok(())
}

97
radixtree/examples/prefix_operations.rs
View File

@@ -1,97 +0,0 @@
use radixtree::RadixTree;
use std::path::PathBuf;
fn main() -> Result<(), radixtree::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("radixtree_prefix_example");
std::fs::create_dir_all(&db_path)?;
println!("Creating radix tree at: {}", db_path.display());
// Create a new radix tree
let mut tree = RadixTree::new(db_path.to_str().unwrap(), true)?;
// Store data with common prefixes
println!("Storing data with common prefixes...");
// User data
tree.set("user:1:name", b"Alice".to_vec())?;
tree.set("user:1:email", b"alice@example.com".to_vec())?;
tree.set("user:2:name", b"Bob".to_vec())?;
tree.set("user:2:email", b"bob@example.com".to_vec())?;
// Post data
tree.set("post:1:title", b"First Post".to_vec())?;
tree.set("post:1:content", b"Hello World!".to_vec())?;
tree.set("post:2:title", b"Second Post".to_vec())?;
tree.set("post:2:content", b"Another post content".to_vec())?;
// Demonstrate listing keys with a prefix
println!("\nListing keys with prefix 'user:1:'");
let user1_keys = tree.list("user:1:")?;
for key in &user1_keys {
println!(" Key: {}", key);
}
println!("\nListing keys with prefix 'post:'");
let post_keys = tree.list("post:")?;
for key in &post_keys {
println!(" Key: {}", key);
}
// Demonstrate getting all values with a prefix
println!("\nGetting all values with prefix 'user:1:'");
let user1_values = tree.getall("user:1:")?;
for (i, value) in user1_values.iter().enumerate() {
println!(" Value {}: {}", i + 1, String::from_utf8_lossy(value));
}
// Demonstrate finding all user names
println!("\nFinding all user names (prefix 'user:*:name')");
let mut user_names = Vec::new();
let all_keys = tree.list("user:")?;
for key in all_keys {
if key.ends_with(":name") {
if let Ok(value) = tree.get(&key) {
user_names.push((key, String::from_utf8_lossy(&value).to_string()));
}
}
}
for (key, name) in user_names {
println!(" {}: {}", key, name);
}
// Demonstrate updating values with a specific prefix
println!("\nUpdating all post titles...");
let post_title_keys = tree.list("post:")?.into_iter().filter(|k| k.ends_with(":title")).collect::<Vec<_>>();
for key in post_title_keys {
let old_value = tree.get(&key)?;
let old_title = String::from_utf8_lossy(&old_value);
let new_title = format!("UPDATED: {}", old_title);
println!(" Updating '{}' to '{}'", old_title, new_title);
tree.update(&key, new_title.as_bytes().to_vec())?;
}
// Verify updates
println!("\nVerifying updates:");
let post_keys = tree.list("post:")?;
for key in post_keys {
if key.ends_with(":title") {
let value = tree.get(&key)?;
println!(" {}: {}", key, String::from_utf8_lossy(&value));
}
}
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("\nCleaned up database directory");
} else {
println!("\nDatabase kept at: {}", db_path.display());
}
Ok(())
}

35
radixtree/src/error.rs
View File

@@ -1,35 +0,0 @@
//! Error types for the RadixTree module.
use thiserror::Error;
/// Error type for RadixTree operations.
#[derive(Debug, Error)]
pub enum Error {
/// Error from OurDB operations.
#[error("OurDB error: {0}")]
OurDB(#[from] ourdb::Error),
/// Error when a key is not found.
#[error("Key not found: {0}")]
KeyNotFound(String),
/// Error when a prefix is not found.
#[error("Prefix not found: {0}")]
PrefixNotFound(String),
/// Error during serialization.
#[error("Serialization error: {0}")]
Serialization(String),
/// Error during deserialization.
#[error("Deserialization error: {0}")]
Deserialization(String),
/// Error for invalid operations.
#[error("Invalid operation: {0}")]
InvalidOperation(String),
/// Error for I/O operations.
#[error("I/O error: {0}")]
IO(#[from] std::io::Error),
}

133
radixtree/src/lib.rs
View File

@@ -1,133 +0,0 @@
//! RadixTree is a space-optimized tree data structure that enables efficient string key operations
//! with persistent storage using OurDB as a backend.
//!
//! This implementation provides a persistent radix tree that can be used for efficient
//! prefix-based key operations, such as auto-complete, routing tables, and more.
mod error;
mod node;
mod operations;
mod serialize;
pub use error::Error;
pub use node::{Node, NodeRef};
use ourdb::OurDB;
/// RadixTree represents a radix tree data structure with persistent storage.
pub struct RadixTree {
db: OurDB,
root_id: u32,
}
impl RadixTree {
/// Creates a new radix tree with the specified database path.
///
/// # Arguments
///
/// * `path` - The path to the database directory
/// * `reset` - Whether to reset the database if it exists
///
/// # Returns
///
/// A new `RadixTree` instance
///
/// # Errors
///
/// Returns an error if the database cannot be created or opened
pub fn new(path: &str, reset: bool) -> Result<Self, Error> {
operations::new_radix_tree(path, reset)
}
/// Sets a key-value pair in the tree.
///
/// # Arguments
///
/// * `key` - The key to set
/// * `value` - The value to set
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn set(&mut self, key: &str, value: Vec<u8>) -> Result<(), Error> {
operations::set(self, key, value)
}
/// Gets a value by key from the tree.
///
/// # Arguments
///
/// * `key` - The key to get
///
/// # Returns
///
/// The value associated with the key
///
/// # Errors
///
/// Returns an error if the key is not found or the operation fails
pub fn get(&mut self, key: &str) -> Result<Vec<u8>, Error> {
operations::get(self, key)
}
/// Updates the value at a given key prefix.
///
/// # Arguments
///
/// * `prefix` - The key prefix to update
/// * `new_value` - The new value to set
///
/// # Errors
///
/// Returns an error if the prefix is not found or the operation fails
pub fn update(&mut self, prefix: &str, new_value: Vec<u8>) -> Result<(), Error> {
operations::update(self, prefix, new_value)
}
/// Deletes a key from the tree.
///
/// # Arguments
///
/// * `key` - The key to delete
///
/// # Errors
///
/// Returns an error if the key is not found or the operation fails
pub fn delete(&mut self, key: &str) -> Result<(), Error> {
operations::delete(self, key)
}
/// Lists all keys with a given prefix.
///
/// # Arguments
///
/// * `prefix` - The prefix to search for
///
/// # Returns
///
/// A list of keys that start with the given prefix
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn list(&mut self, prefix: &str) -> Result<Vec<String>, Error> {
operations::list(self, prefix)
}
/// Gets all values for keys with a given prefix.
///
/// # Arguments
///
/// * `prefix` - The prefix to search for
///
/// # Returns
///
/// A list of values for keys that start with the given prefix
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn getall(&mut self, prefix: &str) -> Result<Vec<Vec<u8>>, Error> {
operations::getall(self, prefix)
}
}

59
radixtree/src/node.rs
View File

@@ -1,59 +0,0 @@
//! Node types for the RadixTree module.
/// Represents a node in the radix tree.
#[derive(Debug, Clone, PartialEq)]
pub struct Node {
/// The segment of the key stored at this node.
pub key_segment: String,
/// Value stored at this node (empty if not a leaf).
pub value: Vec<u8>,
/// References to child nodes.
pub children: Vec<NodeRef>,
/// Whether this node is a leaf node.
pub is_leaf: bool,
}
/// Reference to a node in the database.
#[derive(Debug, Clone, PartialEq)]
pub struct NodeRef {
/// The key segment for this child.
pub key_part: String,
/// Database ID of the node.
pub node_id: u32,
}
impl Node {
/// Creates a new node.
pub fn new(key_segment: String, value: Vec<u8>, is_leaf: bool) -> Self {
Self {
key_segment,
value,
children: Vec::new(),
is_leaf,
}
}
/// Creates a new root node.
pub fn new_root() -> Self {
Self {
key_segment: String::new(),
value: Vec::new(),
children: Vec::new(),
is_leaf: false,
}
}
}
impl NodeRef {
/// Creates a new node reference.
pub fn new(key_part: String, node_id: u32) -> Self {
Self {
key_part,
node_id,
}
}
}

508
radixtree/src/operations.rs
View File

@@ -1,508 +0,0 @@
//! Implementation of RadixTree operations.
use crate::error::Error;
use crate::node::{Node, NodeRef};
use crate::RadixTree;
use crate::serialize::get_common_prefix;
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use std::path::PathBuf;
/// Creates a new radix tree with the specified database path.
pub fn new_radix_tree(path: &str, reset: bool) -> Result<RadixTree, Error> {
let config = OurDBConfig {
path: PathBuf::from(path),
incremental_mode: true,
file_size: Some(1024 * 1024 * 10), // 10MB file size for better performance with large datasets
keysize: Some(6), // Use keysize=6 to support multiple files (file_nr + position)
reset: None, // Don't reset existing database
};
let mut db = OurDB::new(config)?;
// If reset is true, we would clear the database
// Since OurDB doesn't have a reset method, we'll handle it by
// creating a fresh database when reset is true
// We'll implement this by checking if it's a new database (next_id == 1)
let root_id = if db.get_next_id()? == 1 {
// Create a new root node
let root = Node::new_root();
let root_id = db.set(OurDBSetArgs {
id: None,
data: &root.serialize(),
})?;
// First ID should be 1
assert_eq!(root_id, 1);
root_id
} else {
// Use existing root node
1 // Root node always has ID 1
};
Ok(RadixTree {
db,
root_id,
})
}
/// Sets a key-value pair in the tree.
pub fn set(tree: &mut RadixTree, key: &str, value: Vec<u8>) -> Result<(), Error> {
let mut current_id = tree.root_id;
let mut offset = 0;
// Handle empty key case
if key.is_empty() {
let mut root_node = tree.get_node(current_id)?;
root_node.is_leaf = true;
root_node.value = value;
tree.save_node(Some(current_id), &root_node)?;
return Ok(());
}
while offset < key.len() {
let mut node = tree.get_node(current_id)?;
// Find matching child
let mut matched_child = None;
for (i, child) in node.children.iter().enumerate() {
if key[offset..].starts_with(&child.key_part) {
matched_child = Some((i, child.clone()));
break;
}
}
if matched_child.is_none() {
// No matching child found, create new leaf node
let key_part = key[offset..].to_string();
let new_node = Node {
key_segment: key_part.clone(),
value: value.clone(),
children: Vec::new(),
is_leaf: true,
};
let new_id = tree.save_node(None, &new_node)?;
// Create new child reference and update parent node
node.children.push(NodeRef {
key_part,
node_id: new_id,
});
tree.save_node(Some(current_id), &node)?;
return Ok(());
}
let (child_index, mut child) = matched_child.unwrap();
let common_prefix = get_common_prefix(&key[offset..], &child.key_part);
if common_prefix.len() < child.key_part.len() {
// Split existing node
let child_node = tree.get_node(child.node_id)?;
// Create new intermediate node
let new_node = Node {
key_segment: child.key_part[common_prefix.len()..].to_string(),
value: child_node.value.clone(),
children: child_node.children.clone(),
is_leaf: child_node.is_leaf,
};
let new_id = tree.save_node(None, &new_node)?;
// Update current node
node.children[child_index] = NodeRef {
key_part: common_prefix.to_string(),
node_id: new_id,
};
tree.save_node(Some(current_id), &node)?;
// Update child node reference
child.node_id = new_id;
}
if offset + common_prefix.len() == key.len() {
// Update value at existing node
let mut child_node = tree.get_node(child.node_id)?;
child_node.value = value;
child_node.is_leaf = true;
tree.save_node(Some(child.node_id), &child_node)?;
return Ok(());
}
offset += common_prefix.len();
current_id = child.node_id;
}
Ok(())
}
/// Gets a value by key from the tree.
pub fn get(tree: &mut RadixTree, key: &str) -> Result<Vec<u8>, Error> {
let mut current_id = tree.root_id;
let mut offset = 0;
// Handle empty key case
if key.is_empty() {
let root_node = tree.get_node(current_id)?;
if root_node.is_leaf {
return Ok(root_node.value.clone());
}
return Err(Error::KeyNotFound(key.to_string()));
}
while offset < key.len() {
let node = tree.get_node(current_id)?;
let mut found = false;
for child in &node.children {
if key[offset..].starts_with(&child.key_part) {
if offset + child.key_part.len() == key.len() {
let child_node = tree.get_node(child.node_id)?;
if child_node.is_leaf {
return Ok(child_node.value);
}
}
current_id = child.node_id;
offset += child.key_part.len();
found = true;
break;
}
}
if !found {
return Err(Error::KeyNotFound(key.to_string()));
}
}
Err(Error::KeyNotFound(key.to_string()))
}
/// Updates the value at a given key prefix.
pub fn update(tree: &mut RadixTree, prefix: &str, new_value: Vec<u8>) -> Result<(), Error> {
let mut current_id = tree.root_id;
let mut offset = 0;
// Handle empty prefix case
if prefix.is_empty() {
return Err(Error::InvalidOperation("Empty prefix not allowed".to_string()));
}
while offset < prefix.len() {
let node = tree.get_node(current_id)?;
let mut found = false;
for child in &node.children {
if prefix[offset..].starts_with(&child.key_part) {
if offset + child.key_part.len() == prefix.len() {
// Found exact prefix match
let mut child_node = tree.get_node(child.node_id)?;
if child_node.is_leaf {
// Update the value
child_node.value = new_value;
tree.save_node(Some(child.node_id), &child_node)?;
return Ok(());
}
}
current_id = child.node_id;
offset += child.key_part.len();
found = true;
break;
}
}
if !found {
return Err(Error::PrefixNotFound(prefix.to_string()));
}
}
Err(Error::PrefixNotFound(prefix.to_string()))
}
/// Deletes a key from the tree.
pub fn delete(tree: &mut RadixTree, key: &str) -> Result<(), Error> {
let mut current_id = tree.root_id;
let mut offset = 0;
let mut path = Vec::new();
// Handle empty key case
if key.is_empty() {
let mut root_node = tree.get_node(current_id)?;
if !root_node.is_leaf {
return Err(Error::KeyNotFound(key.to_string()));
}
// For the root node, we just mark it as non-leaf
root_node.is_leaf = false;
root_node.value = Vec::new();
tree.save_node(Some(current_id), &root_node)?;
return Ok(());
}
// Find the node to delete
while offset < key.len() {
let node = tree.get_node(current_id)?;
let mut found = false;
for child in &node.children {
if key[offset..].starts_with(&child.key_part) {
path.push(child.clone());
current_id = child.node_id;
offset += child.key_part.len();
found = true;
// Check if we've matched the full key
if offset == key.len() {
let child_node = tree.get_node(child.node_id)?;
if child_node.is_leaf {
found = true;
break;
}
}
break;
}
}
if !found {
return Err(Error::KeyNotFound(key.to_string()));
}
}
if path.is_empty() {
return Err(Error::KeyNotFound(key.to_string()));
}
// Get the node to delete
let mut last_node = tree.get_node(path.last().unwrap().node_id)?;
// If the node has children, just mark it as non-leaf
if !last_node.children.is_empty() {
last_node.is_leaf = false;
last_node.value = Vec::new();
tree.save_node(Some(path.last().unwrap().node_id), &last_node)?;
return Ok(());
}
// If node has no children, remove it from parent
if path.len() > 1 {
let parent_id = path[path.len() - 2].node_id;
let mut parent_node = tree.get_node(parent_id)?;
// Find and remove the child from parent
for i in 0..parent_node.children.len() {
if parent_node.children[i].node_id == path.last().unwrap().node_id {
parent_node.children.remove(i);
break;
}
}
tree.save_node(Some(parent_id), &parent_node)?;
// Delete the node from the database
tree.db.delete(path.last().unwrap().node_id)?;
} else {
// If this is a direct child of the root, just mark it as non-leaf
last_node.is_leaf = false;
last_node.value = Vec::new();
tree.save_node(Some(path.last().unwrap().node_id), &last_node)?;
}
Ok(())
}
/// Lists all keys with a given prefix.
pub fn list(tree: &mut RadixTree, prefix: &str) -> Result<Vec<String>, Error> {
let mut result = Vec::new();
// Handle empty prefix case - will return all keys
if prefix.is_empty() {
collect_all_keys(tree, tree.root_id, "", &mut result)?;
return Ok(result);
}
// Start from the root and find all matching keys
find_keys_with_prefix(tree, tree.root_id, "", prefix, &mut result)?;
Ok(result)
}
/// Helper function to find all keys with a given prefix.
fn find_keys_with_prefix(
tree: &mut RadixTree,
node_id: u32,
current_path: &str,
prefix: &str,
result: &mut Vec<String>,
) -> Result<(), Error> {
let node = tree.get_node(node_id)?;
// If the current path already matches or exceeds the prefix length
if current_path.len() >= prefix.len() {
// Check if the current path starts with the prefix
if current_path.starts_with(prefix) {
// If this is a leaf node, add it to the results
if node.is_leaf {
result.push(current_path.to_string());
}
// Collect all keys from this subtree
for child in &node.children {
let child_path = format!("{}{}", current_path, child.key_part);
find_keys_with_prefix(tree, child.node_id, &child_path, prefix, result)?;
}
}
return Ok(());
}
// Current path is shorter than the prefix, continue searching
for child in &node.children {
let child_path = format!("{}{}", current_path, child.key_part);
// Check if this child's path could potentially match the prefix
if prefix.starts_with(current_path) {
// The prefix starts with the current path, so we need to check if
// the child's key_part matches the next part of the prefix
let prefix_remainder = &prefix[current_path.len()..];
// If the prefix remainder starts with the child's key_part or vice versa
if prefix_remainder.starts_with(&child.key_part)
|| (child.key_part.starts_with(prefix_remainder)
&& child.key_part.len() >= prefix_remainder.len()) {
find_keys_with_prefix(tree, child.node_id, &child_path, prefix, result)?;
}
}
}
Ok(())
}
/// Helper function to recursively collect all keys under a node.
fn collect_all_keys(
tree: &mut RadixTree,
node_id: u32,
current_path: &str,
result: &mut Vec<String>,
) -> Result<(), Error> {
let node = tree.get_node(node_id)?;
// If this node is a leaf, add its path to the result
if node.is_leaf {
result.push(current_path.to_string());
}
// Recursively collect keys from all children
for child in &node.children {
let child_path = format!("{}{}", current_path, child.key_part);
collect_all_keys(tree, child.node_id, &child_path, result)?;
}
Ok(())
}
/// Gets all values for keys with a given prefix.
pub fn getall(tree: &mut RadixTree, prefix: &str) -> Result<Vec<Vec<u8>>, Error> {
// Get all matching keys
let keys = list(tree, prefix)?;
// Get values for each key
let mut values = Vec::new();
for key in keys {
if let Ok(value) = get(tree, &key) {
values.push(value);
}
}
Ok(values)
}
impl RadixTree {
/// Helper function to get a node from the database.
pub(crate) fn get_node(&mut self, node_id: u32) -> Result<Node, Error> {
let data = self.db.get(node_id)?;
Node::deserialize(&data)
}
/// Helper function to save a node to the database.
pub(crate) fn save_node(&mut self, node_id: Option<u32>, node: &Node) -> Result<u32, Error> {
let data = node.serialize();
let args = OurDBSetArgs {
id: node_id,
data: &data,
};
Ok(self.db.set(args)?)
}
/// Helper function to find all keys with a given prefix.
fn find_keys_with_prefix(
&mut self,
node_id: u32,
current_path: &str,
prefix: &str,
result: &mut Vec<String>,
) -> Result<(), Error> {
let node = self.get_node(node_id)?;
// If the current path already matches or exceeds the prefix length
if current_path.len() >= prefix.len() {
// Check if the current path starts with the prefix
if current_path.starts_with(prefix) {
// If this is a leaf node, add it to the results
if node.is_leaf {
result.push(current_path.to_string());
}
// Collect all keys from this subtree
for child in &node.children {
let child_path = format!("{}{}", current_path, child.key_part);
self.find_keys_with_prefix(child.node_id, &child_path, prefix, result)?;
}
}
return Ok(());
}
// Current path is shorter than the prefix, continue searching
for child in &node.children {
let child_path = format!("{}{}", current_path, child.key_part);
// Check if this child's path could potentially match the prefix
if prefix.starts_with(current_path) {
// The prefix starts with the current path, so we need to check if
// the child's key_part matches the next part of the prefix
let prefix_remainder = &prefix[current_path.len()..];
// If the prefix remainder starts with the child's key_part or vice versa
if prefix_remainder.starts_with(&child.key_part)
|| (child.key_part.starts_with(prefix_remainder)
&& child.key_part.len() >= prefix_remainder.len()) {
self.find_keys_with_prefix(child.node_id, &child_path, prefix, result)?;
}
}
}
Ok(())
}
/// Helper function to recursively collect all keys under a node.
fn collect_all_keys(
&mut self,
node_id: u32,
current_path: &str,
result: &mut Vec<String>,
) -> Result<(), Error> {
let node = self.get_node(node_id)?;
// If this node is a leaf, add its path to the result
if node.is_leaf {
result.push(current_path.to_string());
}
// Recursively collect keys from all children
for child in &node.children {
let child_path = format!("{}{}", current_path, child.key_part);
self.collect_all_keys(child.node_id, &child_path, result)?;
}
Ok(())
}
}

156
radixtree/src/serialize.rs
View File

@@ -1,156 +0,0 @@
//! Serialization and deserialization for RadixTree nodes.
use crate::error::Error;
use crate::node::{Node, NodeRef};
use std::io::{Cursor, Read};
use std::mem::size_of;
/// Current binary format version.
const VERSION: u8 = 1;
impl Node {
/// Serializes a node to bytes for storage.
pub fn serialize(&self) -> Vec<u8> {
let mut buffer = Vec::new();
// Add version byte
buffer.push(VERSION);
// Add key segment
write_string(&mut buffer, &self.key_segment);
// Add value as []u8
write_u16(&mut buffer, self.value.len() as u16);
buffer.extend_from_slice(&self.value);
// Add children
write_u16(&mut buffer, self.children.len() as u16);
for child in &self.children {
write_string(&mut buffer, &child.key_part);
write_u32(&mut buffer, child.node_id);
}
// Add leaf flag
buffer.push(if self.is_leaf { 1 } else { 0 });
buffer
}
/// Deserializes bytes to a node.
pub fn deserialize(data: &[u8]) -> Result<Self, Error> {
if data.is_empty() {
return Err(Error::Deserialization("Empty data".to_string()));
}
let mut cursor = Cursor::new(data);
// Read and verify version
let mut version_byte = [0u8; 1];
cursor.read_exact(&mut version_byte)
.map_err(|e| Error::Deserialization(format!("Failed to read version byte: {}", e)))?;
if version_byte[0] != VERSION {
return Err(Error::Deserialization(
format!("Invalid version byte: expected {}, got {}", VERSION, version_byte[0])
));
}
// Read key segment
let key_segment = read_string(&mut cursor)
.map_err(|e| Error::Deserialization(format!("Failed to read key segment: {}", e)))?;
// Read value as []u8
let value_len = read_u16(&mut cursor)
.map_err(|e| Error::Deserialization(format!("Failed to read value length: {}", e)))?;
let mut value = vec![0u8; value_len as usize];
cursor.read_exact(&mut value)
.map_err(|e| Error::Deserialization(format!("Failed to read value: {}", e)))?;
// Read children
let children_len = read_u16(&mut cursor)
.map_err(|e| Error::Deserialization(format!("Failed to read children length: {}", e)))?;
let mut children = Vec::with_capacity(children_len as usize);
for _ in 0..children_len {
let key_part = read_string(&mut cursor)
.map_err(|e| Error::Deserialization(format!("Failed to read child key part: {}", e)))?;
let node_id = read_u32(&mut cursor)
.map_err(|e| Error::Deserialization(format!("Failed to read child node ID: {}", e)))?;
children.push(NodeRef {
key_part,
node_id,
});
}
// Read leaf flag
let mut is_leaf_byte = [0u8; 1];
cursor.read_exact(&mut is_leaf_byte)
.map_err(|e| Error::Deserialization(format!("Failed to read leaf flag: {}", e)))?;
let is_leaf = is_leaf_byte[0] == 1;
Ok(Node {
key_segment,
value,
children,
is_leaf,
})
}
}
// Helper functions for serialization
fn write_string(buffer: &mut Vec<u8>, s: &str) {
let bytes = s.as_bytes();
write_u16(buffer, bytes.len() as u16);
buffer.extend_from_slice(bytes);
}
fn write_u16(buffer: &mut Vec<u8>, value: u16) {
buffer.extend_from_slice(&value.to_le_bytes());
}
fn write_u32(buffer: &mut Vec<u8>, value: u32) {
buffer.extend_from_slice(&value.to_le_bytes());
}
// Helper functions for deserialization
fn read_string(cursor: &mut Cursor<&[u8]>) -> std::io::Result<String> {
let len = read_u16(cursor)? as usize;
let mut bytes = vec![0u8; len];
cursor.read_exact(&mut bytes)?;
String::from_utf8(bytes)
.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))
}
fn read_u16(cursor: &mut Cursor<&[u8]>) -> std::io::Result<u16> {
let mut bytes = [0u8; size_of::<u16>()];
cursor.read_exact(&mut bytes)?;
Ok(u16::from_le_bytes(bytes))
}
fn read_u32(cursor: &mut Cursor<&[u8]>) -> std::io::Result<u32> {
let mut bytes = [0u8; size_of::<u32>()];
cursor.read_exact(&mut bytes)?;
Ok(u32::from_le_bytes(bytes))
}
/// Helper function to get the common prefix of two strings.
pub fn get_common_prefix(a: &str, b: &str) -> String {
let mut i = 0;
let a_bytes = a.as_bytes();
let b_bytes = b.as_bytes();
while i < a.len() && i < b.len() && a_bytes[i] == b_bytes[i] {
i += 1;
}
a[..i].to_string()
}

144
radixtree/tests/basic_test.rs
View File

@@ -1,144 +0,0 @@
use radixtree::RadixTree;
use std::path::PathBuf;
use tempfile::tempdir;
#[test]
fn test_basic_operations() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Test setting and getting values
let key = "test_key";
let value = b"test_value".to_vec();
tree.set(key, value.clone())?;
let retrieved_value = tree.get(key)?;
assert_eq!(retrieved_value, value);
// Test updating a value
let new_value = b"updated_value".to_vec();
tree.update(key, new_value.clone())?;
let updated_value = tree.get(key)?;
assert_eq!(updated_value, new_value);
// Test deleting a value
tree.delete(key)?;
// Trying to get a deleted key should return an error
let result = tree.get(key);
assert!(result.is_err());
Ok(())
}
#[test]
fn test_empty_key() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Test setting and getting empty key
let key = "";
let value = b"value_for_empty_key".to_vec();
tree.set(key, value.clone())?;
let retrieved_value = tree.get(key)?;
assert_eq!(retrieved_value, value);
// Test deleting empty key
tree.delete(key)?;
// Trying to get a deleted key should return an error
let result = tree.get(key);
assert!(result.is_err());
Ok(())
}
#[test]
fn test_multiple_keys() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Insert multiple keys
let test_data = [
("key1", b"value1".to_vec()),
("key2", b"value2".to_vec()),
("key3", b"value3".to_vec()),
];
for (key, value) in &test_data {
tree.set(key, value.clone())?;
}
// Verify all keys can be retrieved
for (key, expected_value) in &test_data {
let retrieved_value = tree.get(key)?;
assert_eq!(&retrieved_value, expected_value);
}
Ok(())
}
#[test]
fn test_shared_prefixes() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Insert keys with shared prefixes
let test_data = [
("test", b"value_test".to_vec()),
("testing", b"value_testing".to_vec()),
("tested", b"value_tested".to_vec()),
];
for (key, value) in &test_data {
tree.set(key, value.clone())?;
}
// Verify all keys can be retrieved
for (key, expected_value) in &test_data {
let retrieved_value = tree.get(key)?;
assert_eq!(&retrieved_value, expected_value);
}
Ok(())
}
#[test]
fn test_persistence() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree and add some data
{
let mut tree = RadixTree::new(db_path, true)?;
tree.set("persistent_key", b"persistent_value".to_vec())?;
} // Tree is dropped here
// Create a new tree instance with the same path
{
let mut tree = RadixTree::new(db_path, false)?;
let value = tree.get("persistent_key")?;
assert_eq!(value, b"persistent_value".to_vec());
}
Ok(())
}

153
radixtree/tests/getall_test.rs
View File

@@ -1,153 +0,0 @@
use radixtree::RadixTree;
use std::collections::HashMap;
use tempfile::tempdir;
#[test]
fn test_getall() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Set up test data with common prefixes
let test_data: HashMap<&str, &str> = [
("user_1", "data1"),
("user_2", "data2"),
("user_3", "data3"),
("admin_1", "admin_data1"),
("admin_2", "admin_data2"),
("guest", "guest_data"),
].iter().cloned().collect();
// Set all test data
for (key, value) in &test_data {
tree.set(key, value.as_bytes().to_vec())?;
}
// Test getall with 'user_' prefix
let user_values = tree.getall("user_")?;
// Should return 3 values
assert_eq!(user_values.len(), 3);
// Convert byte arrays to strings for easier comparison
let user_value_strings: Vec<String> = user_values
.iter()
.map(|v| String::from_utf8_lossy(v).to_string())
.collect();
// Check all expected values are present
assert!(user_value_strings.contains(&"data1".to_string()));
assert!(user_value_strings.contains(&"data2".to_string()));
assert!(user_value_strings.contains(&"data3".to_string()));
// Test getall with 'admin_' prefix
let admin_values = tree.getall("admin_")?;
// Should return 2 values
assert_eq!(admin_values.len(), 2);
// Convert byte arrays to strings for easier comparison
let admin_value_strings: Vec<String> = admin_values
.iter()
.map(|v| String::from_utf8_lossy(v).to_string())
.collect();
// Check all expected values are present
assert!(admin_value_strings.contains(&"admin_data1".to_string()));
assert!(admin_value_strings.contains(&"admin_data2".to_string()));
// Test getall with empty prefix (should return all values)
let all_values = tree.getall("")?;
// Should return all 6 values
assert_eq!(all_values.len(), test_data.len());
// Test getall with non-existent prefix
let non_existent_values = tree.getall("xyz")?;
// Should return empty array
assert_eq!(non_existent_values.len(), 0);
Ok(())
}
#[test]
fn test_getall_with_updates() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Set initial values
tree.set("key1", b"value1".to_vec())?;
tree.set("key2", b"value2".to_vec())?;
tree.set("key3", b"value3".to_vec())?;
// Get initial values
let initial_values = tree.getall("key")?;
assert_eq!(initial_values.len(), 3);
// Update a value
tree.update("key2", b"updated_value2".to_vec())?;
// Get values after update
let updated_values = tree.getall("key")?;
assert_eq!(updated_values.len(), 3);
// Convert to strings for easier comparison
let updated_value_strings: Vec<String> = updated_values
.iter()
.map(|v| String::from_utf8_lossy(v).to_string())
.collect();
// Check the updated value is present
assert!(updated_value_strings.contains(&"value1".to_string()));
assert!(updated_value_strings.contains(&"updated_value2".to_string()));
assert!(updated_value_strings.contains(&"value3".to_string()));
Ok(())
}
#[test]
fn test_getall_with_deletions() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Set initial values
tree.set("prefix_1", b"value1".to_vec())?;
tree.set("prefix_2", b"value2".to_vec())?;
tree.set("prefix_3", b"value3".to_vec())?;
tree.set("other", b"other_value".to_vec())?;
// Get initial values
let initial_values = tree.getall("prefix_")?;
assert_eq!(initial_values.len(), 3);
// Delete a key
tree.delete("prefix_2")?;
// Get values after deletion
let after_delete_values = tree.getall("prefix_")?;
assert_eq!(after_delete_values.len(), 2);
// Convert to strings for easier comparison
let after_delete_strings: Vec<String> = after_delete_values
.iter()
.map(|v| String::from_utf8_lossy(v).to_string())
.collect();
// Check the remaining values
assert!(after_delete_strings.contains(&"value1".to_string()));
assert!(after_delete_strings.contains(&"value3".to_string()));
Ok(())
}

185
radixtree/tests/prefix_test.rs
View File

@@ -1,185 +0,0 @@
use radixtree::RadixTree;
use std::collections::HashMap;
use tempfile::tempdir;
#[test]
fn test_list() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Insert keys with various prefixes
let test_data: HashMap<&str, &str> = [
("apple", "fruit1"),
("application", "software1"),
("apply", "verb1"),
("banana", "fruit2"),
("ball", "toy1"),
("cat", "animal1"),
("car", "vehicle1"),
("cargo", "shipping1"),
].iter().cloned().collect();
// Set all test data
for (key, value) in &test_data {
tree.set(key, value.as_bytes().to_vec())?;
}
// Test prefix 'app' - should return apple, application, apply
let app_keys = tree.list("app")?;
assert_eq!(app_keys.len(), 3);
assert!(app_keys.contains(&"apple".to_string()));
assert!(app_keys.contains(&"application".to_string()));
assert!(app_keys.contains(&"apply".to_string()));
// Test prefix 'ba' - should return banana, ball
let ba_keys = tree.list("ba")?;
assert_eq!(ba_keys.len(), 2);
assert!(ba_keys.contains(&"banana".to_string()));
assert!(ba_keys.contains(&"ball".to_string()));
// Test prefix 'car' - should return car, cargo
let car_keys = tree.list("car")?;
assert_eq!(car_keys.len(), 2);
assert!(car_keys.contains(&"car".to_string()));
assert!(car_keys.contains(&"cargo".to_string()));
// Test prefix 'z' - should return empty list
let z_keys = tree.list("z")?;
assert_eq!(z_keys.len(), 0);
// Test empty prefix - should return all keys
let all_keys = tree.list("")?;
assert_eq!(all_keys.len(), test_data.len());
for key in test_data.keys() {
assert!(all_keys.contains(&key.to_string()));
}
// Test exact key as prefix - should return just that key
let exact_key = tree.list("apple")?;
assert_eq!(exact_key.len(), 1);
assert_eq!(exact_key[0], "apple");
Ok(())
}
#[test]
fn test_list_with_deletion() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Set keys with common prefixes
tree.set("test1", b"value1".to_vec())?;
tree.set("test2", b"value2".to_vec())?;
tree.set("test3", b"value3".to_vec())?;
tree.set("other", b"value4".to_vec())?;
// Initial check
let test_keys = tree.list("test")?;
assert_eq!(test_keys.len(), 3);
assert!(test_keys.contains(&"test1".to_string()));
assert!(test_keys.contains(&"test2".to_string()));
assert!(test_keys.contains(&"test3".to_string()));
// Delete one key
tree.delete("test2")?;
// Check after deletion
let test_keys_after = tree.list("test")?;
assert_eq!(test_keys_after.len(), 2);
assert!(test_keys_after.contains(&"test1".to_string()));
assert!(!test_keys_after.contains(&"test2".to_string()));
assert!(test_keys_after.contains(&"test3".to_string()));
// Check all keys
let all_keys = tree.list("")?;
assert_eq!(all_keys.len(), 3);
assert!(all_keys.contains(&"other".to_string()));
Ok(())
}
#[test]
fn test_list_edge_cases() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Test with empty tree
let empty_result = tree.list("any")?;
assert_eq!(empty_result.len(), 0);
// Set a single key
tree.set("single", b"value".to_vec())?;
// Test with prefix that's longer than any key
let long_prefix = tree.list("singlelonger")?;
assert_eq!(long_prefix.len(), 0);
// Test with partial prefix match
let partial = tree.list("sing")?;
assert_eq!(partial.len(), 1);
assert_eq!(partial[0], "single");
// Test with very long keys
let long_key1 = "a".repeat(100) + "key1";
let long_key2 = "a".repeat(100) + "key2";
tree.set(&long_key1, b"value1".to_vec())?;
tree.set(&long_key2, b"value2".to_vec())?;
let long_prefix_result = tree.list(&"a".repeat(100))?;
assert_eq!(long_prefix_result.len(), 2);
assert!(long_prefix_result.contains(&long_key1));
assert!(long_prefix_result.contains(&long_key2));
Ok(())
}
#[test]
fn test_list_performance() -> Result<(), radixtree::Error> {
// Create a temporary directory for the test
let temp_dir = tempdir().expect("Failed to create temp directory");
let db_path = temp_dir.path().to_str().unwrap();
// Create a new radix tree
let mut tree = RadixTree::new(db_path, true)?;
// Insert a large number of keys with different prefixes
let prefixes = ["user", "post", "comment", "like", "share"];
// Set 100 keys for each prefix (500 total)
for prefix in &prefixes {
for i in 0..100 {
let key = format!("{}_{}", prefix, i);
tree.set(&key, format!("value_{}", key).as_bytes().to_vec())?;
}
}
// Test retrieving by each prefix
for prefix in &prefixes {
let keys = tree.list(prefix)?;
assert_eq!(keys.len(), 100);
// Verify all keys have the correct prefix
for key in &keys {
assert!(key.starts_with(prefix));
}
}
// Test retrieving all keys
let all_keys = tree.list("")?;
assert_eq!(all_keys.len(), 500);
Ok(())
}

180
radixtree/tests/serialize_test.rs
View File

@@ -1,180 +0,0 @@
use radixtree::{Node, NodeRef};
#[test]
fn test_node_serialization() {
// Create a node with some data
let node = Node {
key_segment: "test".to_string(),
value: b"test_value".to_vec(),
children: vec![
NodeRef {
key_part: "child1".to_string(),
node_id: 1,
},
NodeRef {
key_part: "child2".to_string(),
node_id: 2,
},
],
is_leaf: true,
};
// Serialize the node
let serialized = node.serialize();
// Deserialize the node
let deserialized = Node::deserialize(&serialized).expect("Failed to deserialize node");
// Verify the deserialized node matches the original
assert_eq!(deserialized.key_segment, node.key_segment);
assert_eq!(deserialized.value, node.value);
assert_eq!(deserialized.is_leaf, node.is_leaf);
assert_eq!(deserialized.children.len(), node.children.len());
for (i, child) in node.children.iter().enumerate() {
assert_eq!(deserialized.children[i].key_part, child.key_part);
assert_eq!(deserialized.children[i].node_id, child.node_id);
}
}
#[test]
fn test_empty_node_serialization() {
// Create an empty node
let node = Node {
key_segment: "".to_string(),
value: vec![],
children: vec![],
is_leaf: false,
};
// Serialize the node
let serialized = node.serialize();
// Deserialize the node
let deserialized = Node::deserialize(&serialized).expect("Failed to deserialize node");
// Verify the deserialized node matches the original
assert_eq!(deserialized.key_segment, node.key_segment);
assert_eq!(deserialized.value, node.value);
assert_eq!(deserialized.is_leaf, node.is_leaf);
assert_eq!(deserialized.children.len(), node.children.len());
}
#[test]
fn test_node_with_many_children() {
// Create a node with many children
let mut children = Vec::new();
for i in 0..100 {
children.push(NodeRef {
key_part: format!("child{}", i),
node_id: i as u32,
});
}
let node = Node {
key_segment: "parent".to_string(),
value: b"parent_value".to_vec(),
children,
is_leaf: true,
};
// Serialize the node
let serialized = node.serialize();
// Deserialize the node
let deserialized = Node::deserialize(&serialized).expect("Failed to deserialize node");
// Verify the deserialized node matches the original
assert_eq!(deserialized.key_segment, node.key_segment);
assert_eq!(deserialized.value, node.value);
assert_eq!(deserialized.is_leaf, node.is_leaf);
assert_eq!(deserialized.children.len(), node.children.len());
for (i, child) in node.children.iter().enumerate() {
assert_eq!(deserialized.children[i].key_part, child.key_part);
assert_eq!(deserialized.children[i].node_id, child.node_id);
}
}
#[test]
fn test_node_with_large_value() {
// Create a node with a large value
let large_value = vec![0u8; 4096]; // 4KB value
let node = Node {
key_segment: "large_value".to_string(),
value: large_value.clone(),
children: vec![],
is_leaf: true,
};
// Serialize the node
let serialized = node.serialize();
// Deserialize the node
let deserialized = Node::deserialize(&serialized).expect("Failed to deserialize node");
// Verify the deserialized node matches the original
assert_eq!(deserialized.key_segment, node.key_segment);
assert_eq!(deserialized.value, node.value);
assert_eq!(deserialized.is_leaf, node.is_leaf);
assert_eq!(deserialized.children.len(), node.children.len());
}
#[test]
fn test_version_compatibility() {
// This test ensures that the serialization format is compatible with version 1
// Create a node
let node = Node {
key_segment: "test".to_string(),
value: b"test_value".to_vec(),
children: vec![
NodeRef {
key_part: "child".to_string(),
node_id: 1,
},
],
is_leaf: true,
};
// Serialize the node
let serialized = node.serialize();
// Verify the first byte is the version byte (1)
assert_eq!(serialized[0], 1);
// Deserialize the node
let deserialized = Node::deserialize(&serialized).expect("Failed to deserialize node");
// Verify the deserialized node matches the original
assert_eq!(deserialized.key_segment, node.key_segment);
assert_eq!(deserialized.value, node.value);
assert_eq!(deserialized.is_leaf, node.is_leaf);
assert_eq!(deserialized.children.len(), node.children.len());
}
#[test]
fn test_invalid_serialization() {
// Test with empty data
let result = Node::deserialize(&[]);
assert!(result.is_err());
// Test with invalid version
let result = Node::deserialize(&[2, 0, 0, 0, 0]);
assert!(result.is_err());
// Test with truncated data
let node = Node {
key_segment: "test".to_string(),
value: b"test_value".to_vec(),
children: vec![],
is_leaf: true,
};
let serialized = node.serialize();
let truncated = &serialized[0..serialized.len() / 2];
let result = Node::deserialize(truncated);
assert!(result.is_err());
}

179
tst/Cargo.lock generated
View File

@@ -1,179 +0,0 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
version = 4
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "crc32fast"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]]
name = "getrandom"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if",
"libc",
"wasi",
]
[[package]]
name = "libc"
version = "0.2.172"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
[[package]]
name = "log"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
[[package]]
name = "ourdb"
version = "0.1.0"
dependencies = [
"crc32fast",
"log",
"rand",
"thiserror",
]
[[package]]
name = "ppv-lite86"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
dependencies = [
"zerocopy",
]
[[package]]
name = "proc-macro2"
version = "1.0.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
dependencies = [
"unicode-ident",
]
[[package]]
name = "quote"
version = "1.0.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d"
dependencies = [
"proc-macro2",
]
[[package]]
name = "rand"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
"rand_chacha",
"rand_core",
]
[[package]]
name = "rand_chacha"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
"rand_core",
]
[[package]]
name = "rand_core"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
"getrandom",
]
[[package]]
name = "syn"
version = "2.0.100"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tst"
version = "0.1.0"
dependencies = [
"ourdb",
"thiserror",
]
[[package]]
name = "unicode-ident"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512"
[[package]]
name = "wasi"
version = "0.11.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "zerocopy"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be"
dependencies = [
"proc-macro2",
"quote",
"syn",
]

30
tst/Cargo.toml
View File

@@ -1,30 +0,0 @@
[package]
name = "tst"
version = "0.1.0"
edition = "2021"
description = "A persistent ternary search tree implementation using OurDB for storage"
authors = ["OurWorld Team"]
[dependencies]
ourdb = { path = "../ourdb" }
thiserror = "1.0.40"
[dev-dependencies]
# criterion = "0.5.1"
# Uncomment when benchmarks are implemented
# [[bench]]
# name = "tst_benchmarks"
# harness = false
[[example]]
name = "basic_usage"
path = "examples/basic_usage.rs"
[[example]]
name = "prefix_ops"
path = "examples/prefix_ops.rs"
[[example]]
name = "performance"
path = "examples/performance.rs"

185
tst/README.md
View File

@@ -1,185 +0,0 @@
# Ternary Search Tree (TST)
A persistent ternary search tree implementation in Rust using OurDB for storage.
## Overview
TST is a space-optimized tree data structure that enables efficient string key operations with persistent storage. This implementation provides a persistent ternary search tree that can be used for efficient string key operations, such as auto-complete, routing tables, and more.
A ternary search tree is a type of trie where each node has three children: left, middle, and right. Unlike a radix tree which compresses common prefixes, a TST stores one character per node and uses a binary search tree-like structure for efficient traversal.
Key characteristics:
- Each node stores a single character
- Nodes have three children: left (for characters < current), middle (for next character in key), and right (for characters > current)
- Leaf nodes contain the actual values
- Balanced structure for consistent performance across operations
## Features
- Efficient string key operations
- Persistent storage using OurDB backend
- Balanced tree structure for consistent performance
- Support for binary values
- Thread-safe operations through OurDB
## Usage
Add the dependency to your `Cargo.toml`:
```toml
[dependencies]
tst = { path = "../tst" }
```
### Basic Example
```rust
use tst::TST;
fn main() -> Result<(), tst::Error> {
// Create a new ternary search tree
let mut tree = TST::new("/tmp/tst", false)?;
// Set key-value pairs
tree.set("hello", b"world".to_vec())?;
tree.set("help", b"me".to_vec())?;
// Get values by key
let value = tree.get("hello")?;
println!("hello: {}", String::from_utf8_lossy(&value)); // Prints: world
// List keys by prefix
let keys = tree.list("hel")?; // Returns ["hello", "help"]
println!("Keys with prefix 'hel': {:?}", keys);
// Get all values by prefix
let values = tree.getall("hel")?; // Returns [b"world", b"me"]
// Delete keys
tree.delete("help")?;
Ok(())
}
```
## API
### Creating a TST
```rust
// Create a new ternary search tree
let mut tree = TST::new("/tmp/tst", false)?;
// Create a new ternary search tree and reset if it exists
let mut tree = TST::new("/tmp/tst", true)?;
```
### Setting Values
```rust
// Set a key-value pair
tree.set("key", b"value".to_vec())?;
```
### Getting Values
```rust
// Get a value by key
let value = tree.get("key")?;
```
### Deleting Keys
```rust
// Delete a key
tree.delete("key")?;
```
### Listing Keys by Prefix
```rust
// List all keys with a given prefix
let keys = tree.list("prefix")?;
```
### Getting All Values by Prefix
```rust
// Get all values for keys with a given prefix
let values = tree.getall("prefix")?;
```
## Performance Characteristics
- Search: O(k) where k is the key length
- Insert: O(k) for new keys
- Delete: O(k) plus potential node cleanup
- Space: O(n) where n is the total number of nodes
## Use Cases
TST is particularly useful for:
- Prefix-based searching
- Auto-complete systems
- Dictionary implementations
- Spell checking
- Any application requiring efficient string key operations with persistence
## Implementation Details
The TST implementation uses OurDB for persistent storage:
- Each node is serialized and stored as a record in OurDB
- Node references use OurDB record IDs
- The tree maintains a root node ID for traversal
- Node serialization includes version tracking for format evolution
## Running Tests
The project includes a comprehensive test suite that verifies all functionality:
```bash
cd ~/code/git.threefold.info/herocode/db/tst
# Run all tests
cargo test
# Run specific test file
cargo test --test basic_test
cargo test --test prefix_test
```
## Running Examples
The project includes example applications that demonstrate how to use the TST:
```bash
# Run the basic usage example
cargo run --example basic_usage
# Run the prefix operations example
cargo run --example prefix_ops
# Run the performance test
cargo run --example performance
```
## Comparison with RadixTree
While both TST and RadixTree provide efficient string key operations, they have different characteristics:
- **TST**: Stores one character per node, with a balanced structure for consistent performance across operations.
- **RadixTree**: Compresses common prefixes, which can be more space-efficient for keys with long common prefixes.
Choose TST when:
- You need balanced performance across all operations
- Your keys don't share long common prefixes
- You want a simpler implementation with predictable performance
Choose RadixTree when:
- Space efficiency is a priority
- Your keys share long common prefixes
- You prioritize lookup performance over balanced performance
## License
This project is licensed under the same license as the HeroCode project.

75
tst/examples/basic_usage.rs
View File

@@ -1,75 +0,0 @@
use std::time::Instant;
use tst::TST;
fn main() -> Result<(), tst::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("tst_example");
std::fs::create_dir_all(&db_path)?;
println!("Creating ternary search tree at: {}", db_path.display());
// Create a new TST
let mut tree = TST::new(db_path.to_str().unwrap(), true)?;
// Store some data
println!("Inserting data...");
tree.set("hello", b"world".to_vec())?;
tree.set("help", b"me".to_vec())?;
tree.set("helicopter", b"flying".to_vec())?;
tree.set("apple", b"fruit".to_vec())?;
tree.set("application", b"software".to_vec())?;
tree.set("banana", b"yellow".to_vec())?;
// Retrieve and print the data
let value = tree.get("hello")?;
println!("hello: {}", String::from_utf8_lossy(&value));
// List keys with prefix
println!("\nListing keys with prefix 'hel':");
let start = Instant::now();
let keys = tree.list("hel")?;
let duration = start.elapsed();
for key in &keys {
println!(" {}", key);
}
println!("Found {} keys in {:?}", keys.len(), duration);
// Get all values with prefix
println!("\nGetting all values with prefix 'app':");
let start = Instant::now();
let values = tree.getall("app")?;
let duration = start.elapsed();
for (i, value) in values.iter().enumerate() {
println!(" Value {}: {}", i + 1, String::from_utf8_lossy(value));
}
println!("Found {} values in {:?}", values.len(), duration);
// Delete a key
println!("\nDeleting 'help'...");
tree.delete("help")?;
// Verify deletion
println!("Listing keys with prefix 'hel' after deletion:");
let keys_after = tree.list("hel")?;
for key in &keys_after {
println!(" {}", key);
}
// Try to get a deleted key
match tree.get("help") {
Ok(_) => println!("Unexpectedly found 'help' after deletion!"),
Err(e) => println!("As expected, 'help' was not found: {}", e),
}
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("\nCleaned up database directory");
} else {
println!("\nDatabase kept at: {}", db_path.display());
}
Ok(())
}

167
tst/examples/performance.rs
View File

@@ -1,167 +0,0 @@
use std::io::{self, Write};
use std::time::{Duration, Instant};
use tst::TST;
// Function to generate a test value of specified size
fn generate_test_value(index: usize, size: usize) -> Vec<u8> {
let base_value = format!("val{:08}", index);
let mut value = Vec::with_capacity(size);
// Fill with repeating pattern to reach desired size
while value.len() < size {
value.extend_from_slice(base_value.as_bytes());
}
// Truncate to exact size
value.truncate(size);
value
}
// Number of records to insert
const TOTAL_RECORDS: usize = 100_000;
// How often to report progress (every X records)
const PROGRESS_INTERVAL: usize = 1_000;
// How many records to use for performance sampling
const PERFORMANCE_SAMPLE_SIZE: usize = 100;
fn main() -> Result<(), tst::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("tst_performance_test");
// Completely remove and recreate the directory to ensure a clean start
if db_path.exists() {
std::fs::remove_dir_all(&db_path)?;
}
std::fs::create_dir_all(&db_path)?;
println!("Creating ternary search tree at: {}", db_path.display());
println!("Will insert {} records and show progress...", TOTAL_RECORDS);
// Create a new TST
let mut tree = TST::new(db_path.to_str().unwrap(), true)?;
// Track overall time
let start_time = Instant::now();
// Track performance metrics
let mut insertion_times = Vec::with_capacity(TOTAL_RECORDS / PROGRESS_INTERVAL);
let mut last_batch_time = Instant::now();
let mut last_batch_records = 0;
// Insert records and track progress
for i in 0..TOTAL_RECORDS {
let key = format!("key:{:08}", i);
// Generate a 100-byte value
let value = generate_test_value(i, 100);
// Time the insertion of every Nth record for performance sampling
if i % PERFORMANCE_SAMPLE_SIZE == 0 {
let insert_start = Instant::now();
tree.set(&key, value)?;
let insert_duration = insert_start.elapsed();
// Only print detailed timing for specific samples to avoid flooding output
if i % (PERFORMANCE_SAMPLE_SIZE * 10) == 0 {
println!("Record {}: Insertion took {:?}", i, insert_duration);
}
} else {
tree.set(&key, value)?;
}
// Show progress at intervals
if (i + 1) % PROGRESS_INTERVAL == 0 || i == TOTAL_RECORDS - 1 {
let records_in_batch = i + 1 - last_batch_records;
let batch_duration = last_batch_time.elapsed();
let records_per_second = records_in_batch as f64 / batch_duration.as_secs_f64();
insertion_times.push((i + 1, batch_duration));
print!(
"\rProgress: {}/{} records ({:.2}%) - {:.2} records/sec",
i + 1,
TOTAL_RECORDS,
(i + 1) as f64 / TOTAL_RECORDS as f64 * 100.0,
records_per_second
);
io::stdout().flush().unwrap();
last_batch_time = Instant::now();
last_batch_records = i + 1;
}
}
let total_duration = start_time.elapsed();
println!("\n\nPerformance Summary:");
println!(
"Total time to insert {} records: {:?}",
TOTAL_RECORDS, total_duration
);
println!(
"Average insertion rate: {:.2} records/second",
TOTAL_RECORDS as f64 / total_duration.as_secs_f64()
);
// Show performance trend
println!("\nPerformance Trend (records inserted vs. time per batch):");
for (i, (record_count, duration)) in insertion_times.iter().enumerate() {
if i % 10 == 0 || i == insertion_times.len() - 1 {
// Only show every 10th point to avoid too much output
println!(
" After {} records: {:?} for {} records ({:.2} records/sec)",
record_count,
duration,
PROGRESS_INTERVAL,
PROGRESS_INTERVAL as f64 / duration.as_secs_f64()
);
}
}
// Test access performance with distributed samples
println!("\nTesting access performance with distributed samples...");
let mut total_get_time = Duration::new(0, 0);
let num_samples = 1000;
// Use a simple distribution pattern instead of random
for i in 0..num_samples {
// Distribute samples across the entire range
let sample_id = (i * (TOTAL_RECORDS / num_samples)) % TOTAL_RECORDS;
let key = format!("key:{:08}", sample_id);
let get_start = Instant::now();
let _ = tree.get(&key)?;
total_get_time += get_start.elapsed();
}
println!(
"Average time to retrieve a record: {:?}",
total_get_time / num_samples as u32
);
// Test prefix search performance
println!("\nTesting prefix search performance...");
let prefixes = ["key:0", "key:1", "key:5", "key:9"];
for prefix in &prefixes {
let list_start = Instant::now();
let keys = tree.list(prefix)?;
let list_duration = list_start.elapsed();
println!(
"Found {} keys with prefix '{}' in {:?}",
keys.len(),
prefix,
list_duration
);
}
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("\nCleaned up database directory");
} else {
println!("\nDatabase kept at: {}", db_path.display());
}
Ok(())
}

184
tst/examples/prefix_ops.rs
View File

@@ -1,184 +0,0 @@
use std::time::Instant;
use tst::TST;
fn main() -> Result<(), tst::Error> {
// Create a temporary directory for the database
let db_path = std::env::temp_dir().join("tst_prefix_example");
std::fs::create_dir_all(&db_path)?;
println!("Creating ternary search tree at: {}", db_path.display());
// Create a new TST
let mut tree = TST::new(db_path.to_str().unwrap(), true)?;
// Insert a variety of keys with different prefixes
println!("Inserting data with various prefixes...");
// Names
let names = [
"Alice",
"Alexander",
"Amanda",
"Andrew",
"Amy",
"Bob",
"Barbara",
"Benjamin",
"Brenda",
"Brian",
"Charlie",
"Catherine",
"Christopher",
"Cynthia",
"Carl",
"David",
"Diana",
"Daniel",
"Deborah",
"Donald",
"Edward",
"Elizabeth",
"Eric",
"Emily",
"Ethan",
];
for (i, name) in names.iter().enumerate() {
let value = format!("person-{}", i).into_bytes();
tree.set(name, value)?;
}
// Cities
let cities = [
"New York",
"Los Angeles",
"Chicago",
"Houston",
"Phoenix",
"Philadelphia",
"San Antonio",
"San Diego",
"Dallas",
"San Jose",
"Austin",
"Jacksonville",
"Fort Worth",
"Columbus",
"San Francisco",
"Charlotte",
"Indianapolis",
"Seattle",
"Denver",
"Washington",
];
for (i, city) in cities.iter().enumerate() {
let value = format!("city-{}", i).into_bytes();
tree.set(city, value)?;
}
// Countries
let countries = [
"United States",
"Canada",
"Mexico",
"Brazil",
"Argentina",
"United Kingdom",
"France",
"Germany",
"Italy",
"Spain",
"China",
"Japan",
"India",
"Australia",
"Russia",
];
for (i, country) in countries.iter().enumerate() {
let value = format!("country-{}", i).into_bytes();
tree.set(country, value)?;
}
println!(
"Total items inserted: {}",
names.len() + cities.len() + countries.len()
);
// Test prefix operations
test_prefix(&mut tree, "A")?;
test_prefix(&mut tree, "B")?;
test_prefix(&mut tree, "C")?;
test_prefix(&mut tree, "San")?;
test_prefix(&mut tree, "United")?;
// Test non-existent prefix
test_prefix(&mut tree, "Z")?;
// Test empty prefix (should return all keys)
println!("\nTesting empty prefix (should return all keys):");
let start = Instant::now();
let all_keys = tree.list("")?;
let duration = start.elapsed();
println!(
"Found {} keys with empty prefix in {:?}",
all_keys.len(),
duration
);
println!("First 5 keys (alphabetically):");
for key in all_keys.iter().take(5) {
println!(" {}", key);
}
// Clean up (optional)
if std::env::var("KEEP_DB").is_err() {
std::fs::remove_dir_all(&db_path)?;
println!("\nCleaned up database directory");
} else {
println!("\nDatabase kept at: {}", db_path.display());
}
Ok(())
}
fn test_prefix(tree: &mut TST, prefix: &str) -> Result<(), tst::Error> {
println!("\nTesting prefix '{}':", prefix);
// Test list operation
let start = Instant::now();
let keys = tree.list(prefix)?;
let list_duration = start.elapsed();
println!(
"Found {} keys with prefix '{}' in {:?}",
keys.len(),
prefix,
list_duration
);
if !keys.is_empty() {
println!("Keys:");
for key in &keys {
println!(" {}", key);
}
// Test getall operation
let start = Instant::now();
let values = tree.getall(prefix)?;
let getall_duration = start.elapsed();
println!("Retrieved {} values in {:?}", values.len(), getall_duration);
println!(
"First value: {}",
if !values.is_empty() {
String::from_utf8_lossy(&values[0])
} else {
"None".into()
}
);
}
Ok(())
}

36
tst/src/error.rs
View File

@@ -1,36 +0,0 @@
//! Error types for the TST module.
use std::io;
use thiserror::Error;
/// Error type for TST operations.
#[derive(Debug, Error)]
pub enum Error {
/// Error from OurDB operations.
#[error("OurDB error: {0}")]
OurDB(#[from] ourdb::Error),
/// Error when a key is not found.
#[error("Key not found: {0}")]
KeyNotFound(String),
/// Error when a prefix is not found.
#[error("Prefix not found: {0}")]
PrefixNotFound(String),
/// Error during serialization.
#[error("Serialization error: {0}")]
Serialization(String),
/// Error during deserialization.
#[error("Deserialization error: {0}")]
Deserialization(String),
/// Error for invalid operations.
#[error("Invalid operation: {0}")]
InvalidOperation(String),
/// IO error.
#[error("IO error: {0}")]
IO(#[from] io::Error),
}

122
tst/src/lib.rs
View File

@@ -1,122 +0,0 @@
//! TST is a space-optimized tree data structure that enables efficient string key operations
//! with persistent storage using OurDB as a backend.
//!
//! This implementation provides a persistent ternary search tree that can be used for efficient
//! string key operations, such as auto-complete, routing tables, and more.
mod error;
mod node;
mod operations;
mod serialize;
pub use error::Error;
pub use node::TSTNode;
use ourdb::OurDB;
/// TST represents a ternary search tree data structure with persistent storage.
pub struct TST {
/// Database for persistent storage
db: OurDB,
/// Database ID of the root node
root_id: Option<u32>,
}
impl TST {
/// Creates a new TST with the specified database path.
///
/// # Arguments
///
/// * `path` - The path to the database directory
/// * `reset` - Whether to reset the database if it exists
///
/// # Returns
///
/// A new `TST` instance
///
/// # Errors
///
/// Returns an error if the database cannot be created or opened
pub fn new(path: &str, reset: bool) -> Result<Self, Error> {
operations::new_tst(path, reset)
}
/// Sets a key-value pair in the tree.
///
/// # Arguments
///
/// * `key` - The key to set
/// * `value` - The value to set
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn set(&mut self, key: &str, value: Vec<u8>) -> Result<(), Error> {
operations::set(self, key, value)
}
/// Gets a value by key from the tree.
///
/// # Arguments
///
/// * `key` - The key to get
///
/// # Returns
///
/// The value associated with the key
///
/// # Errors
///
/// Returns an error if the key is not found or the operation fails
pub fn get(&mut self, key: &str) -> Result<Vec<u8>, Error> {
operations::get(self, key)
}
/// Deletes a key from the tree.
///
/// # Arguments
///
/// * `key` - The key to delete
///
/// # Errors
///
/// Returns an error if the key is not found or the operation fails
pub fn delete(&mut self, key: &str) -> Result<(), Error> {
operations::delete(self, key)
}
/// Lists all keys with a given prefix.
///
/// # Arguments
///
/// * `prefix` - The prefix to search for
///
/// # Returns
///
/// A list of keys that start with the given prefix
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn list(&mut self, prefix: &str) -> Result<Vec<String>, Error> {
operations::list(self, prefix)
}
/// Gets all values for keys with a given prefix.
///
/// # Arguments
///
/// * `prefix` - The prefix to search for
///
/// # Returns
///
/// A list of values for keys that start with the given prefix
///
/// # Errors
///
/// Returns an error if the operation fails
pub fn getall(&mut self, prefix: &str) -> Result<Vec<Vec<u8>>, Error> {
operations::getall(self, prefix)
}
}

49
tst/src/node.rs
View File

@@ -1,49 +0,0 @@
//! Node types for the TST module.
/// Represents a node in the ternary search tree.
#[derive(Debug, Clone, PartialEq)]
pub struct TSTNode {
/// The character stored at this node.
pub character: char,
/// Value stored at this node (empty if not end of key).
pub value: Vec<u8>,
/// Whether this node represents the end of a key.
pub is_end_of_key: bool,
/// Reference to the left child node (for characters < current character).
pub left_id: Option<u32>,
/// Reference to the middle child node (for next character in key).
pub middle_id: Option<u32>,
/// Reference to the right child node (for characters > current character).
pub right_id: Option<u32>,
}
impl TSTNode {
/// Creates a new node.
pub fn new(character: char, value: Vec<u8>, is_end_of_key: bool) -> Self {
Self {
character,
value,
is_end_of_key,
left_id: None,
middle_id: None,
right_id: None,
}
}
/// Creates a new root node.
pub fn new_root() -> Self {
Self {
character: '\0', // Use null character for root
value: Vec::new(),
is_end_of_key: false,
left_id: None,
middle_id: None,
right_id: None,
}
}
}

453
tst/src/operations.rs
View File

@@ -1,453 +0,0 @@
//! Implementation of TST operations.
use crate::error::Error;
use crate::node::TSTNode;
use crate::TST;
use ourdb::{OurDB, OurDBConfig, OurDBSetArgs};
use std::path::PathBuf;
/// Creates a new TST with the specified database path.
pub fn new_tst(path: &str, reset: bool) -> Result<TST, Error> {
let path_buf = PathBuf::from(path);
// Create the configuration for OurDB with reset parameter
let config = OurDBConfig {
path: path_buf.clone(),
incremental_mode: true,
file_size: Some(1024 * 1024), // 1MB file size for better performance with large datasets
keysize: Some(4), // Use keysize=4 (default)
reset: Some(reset), // Use the reset parameter
};
// Create a new OurDB instance (it will handle reset internally)
let mut db = OurDB::new(config)?;
let root_id = if db.get_next_id()? == 1 || reset {
// Create a new root node
let root = TSTNode::new_root();
let root_id = db.set(OurDBSetArgs {
id: None,
data: &root.serialize(),
})?;
Some(root_id)
} else {
// Use existing root node
Some(1) // Root node always has ID 1
};
Ok(TST { db, root_id })
}
/// Sets a key-value pair in the tree.
pub fn set(tree: &mut TST, key: &str, value: Vec<u8>) -> Result<(), Error> {
if key.is_empty() {
return Err(Error::InvalidOperation("Empty key not allowed".to_string()));
}
let root_id = match tree.root_id {
Some(id) => id,
None => return Err(Error::InvalidOperation("Tree not initialized".to_string())),
};
let chars: Vec<char> = key.chars().collect();
set_recursive(tree, root_id, &chars, 0, value)?;
Ok(())
}
/// Recursive helper function for setting a key-value pair.
fn set_recursive(
tree: &mut TST,
node_id: u32,
chars: &[char],
pos: usize,
value: Vec<u8>,
) -> Result<u32, Error> {
let mut node = tree.get_node(node_id)?;
if pos >= chars.len() {
// We've reached the end of the key
node.is_end_of_key = true;
node.value = value;
return tree.save_node(Some(node_id), &node);
}
let current_char = chars[pos];
if node.character == '\0' {
// Root node or empty node, set the character
node.character = current_char;
let node_id = tree.save_node(Some(node_id), &node)?;
// Continue with the next character
if pos + 1 < chars.len() {
let new_node = TSTNode::new(chars[pos + 1], Vec::new(), false);
let new_id = tree.save_node(None, &new_node)?;
let mut updated_node = tree.get_node(node_id)?;
updated_node.middle_id = Some(new_id);
tree.save_node(Some(node_id), &updated_node)?;
return set_recursive(tree, new_id, chars, pos + 1, value);
} else {
// This is the last character
let mut updated_node = tree.get_node(node_id)?;
updated_node.is_end_of_key = true;
updated_node.value = value;
return tree.save_node(Some(node_id), &updated_node);
}
}
if current_char < node.character {
// Go left
if let Some(left_id) = node.left_id {
return set_recursive(tree, left_id, chars, pos, value);
} else {
// Create new left node
let new_node = TSTNode::new(current_char, Vec::new(), false);
let new_id = tree.save_node(None, &new_node)?;
// Update current node
node.left_id = Some(new_id);
tree.save_node(Some(node_id), &node)?;
return set_recursive(tree, new_id, chars, pos, value);
}
} else if current_char > node.character {
// Go right
if let Some(right_id) = node.right_id {
return set_recursive(tree, right_id, chars, pos, value);
} else {
// Create new right node
let new_node = TSTNode::new(current_char, Vec::new(), false);
let new_id = tree.save_node(None, &new_node)?;
// Update current node
node.right_id = Some(new_id);
tree.save_node(Some(node_id), &node)?;
return set_recursive(tree, new_id, chars, pos, value);
}
} else {
// Character matches, go middle (next character)
if pos + 1 >= chars.len() {
// This is the last character
node.is_end_of_key = true;
node.value = value;
return tree.save_node(Some(node_id), &node);
}
if let Some(middle_id) = node.middle_id {
return set_recursive(tree, middle_id, chars, pos + 1, value);
} else {
// Create new middle node
let new_node = TSTNode::new(chars[pos + 1], Vec::new(), false);
let new_id = tree.save_node(None, &new_node)?;
// Update current node
node.middle_id = Some(new_id);
tree.save_node(Some(node_id), &node)?;
return set_recursive(tree, new_id, chars, pos + 1, value);
}
}
}
/// Gets a value by key from the tree.
pub fn get(tree: &mut TST, key: &str) -> Result<Vec<u8>, Error> {
if key.is_empty() {
return Err(Error::InvalidOperation("Empty key not allowed".to_string()));
}
let root_id = match tree.root_id {
Some(id) => id,
None => return Err(Error::InvalidOperation("Tree not initialized".to_string())),
};
let chars: Vec<char> = key.chars().collect();
let node_id = find_node(tree, root_id, &chars, 0)?;
let node = tree.get_node(node_id)?;
if node.is_end_of_key {
Ok(node.value.clone())
} else {
Err(Error::KeyNotFound(key.to_string()))
}
}
/// Finds a node by key.
fn find_node(tree: &mut TST, node_id: u32, chars: &[char], pos: usize) -> Result<u32, Error> {
let node = tree.get_node(node_id)?;
if pos >= chars.len() {
return Ok(node_id);
}
let current_char = chars[pos];
if current_char < node.character {
// Go left
if let Some(left_id) = node.left_id {
find_node(tree, left_id, chars, pos)
} else {
Err(Error::KeyNotFound(chars.iter().collect()))
}
} else if current_char > node.character {
// Go right
if let Some(right_id) = node.right_id {
find_node(tree, right_id, chars, pos)
} else {
Err(Error::KeyNotFound(chars.iter().collect()))
}
} else {
// Character matches
if pos + 1 >= chars.len() {
// This is the last character
Ok(node_id)
} else if let Some(middle_id) = node.middle_id {
// Go to next character
find_node(tree, middle_id, chars, pos + 1)
} else {
Err(Error::KeyNotFound(chars.iter().collect()))
}
}
}
/// Deletes a key from the tree.
pub fn delete(tree: &mut TST, key: &str) -> Result<(), Error> {
if key.is_empty() {
return Err(Error::InvalidOperation("Empty key not allowed".to_string()));
}
let root_id = match tree.root_id {
Some(id) => id,
None => return Err(Error::InvalidOperation("Tree not initialized".to_string())),
};
let chars: Vec<char> = key.chars().collect();
let node_id = find_node(tree, root_id, &chars, 0)?;
let mut node = tree.get_node(node_id)?;
if !node.is_end_of_key {
return Err(Error::KeyNotFound(key.to_string()));
}
// If the node has a middle child, just mark it as not end of key
if node.middle_id.is_some() || node.left_id.is_some() || node.right_id.is_some() {
node.is_end_of_key = false;
node.value = Vec::new();
tree.save_node(Some(node_id), &node)?;
return Ok(());
}
// Otherwise, we need to remove the node and update its parent
// This is more complex and would require tracking the path to the node
// For simplicity, we'll just mark it as not end of key for now
node.is_end_of_key = false;
node.value = Vec::new();
tree.save_node(Some(node_id), &node)?;
Ok(())
}
/// Lists all keys with a given prefix.
pub fn list(tree: &mut TST, prefix: &str) -> Result<Vec<String>, Error> {
let root_id = match tree.root_id {
Some(id) => id,
None => return Err(Error::InvalidOperation("Tree not initialized".to_string())),
};
let mut result = Vec::new();
// Handle empty prefix case - will return all keys
if prefix.is_empty() {
collect_all_keys(tree, root_id, String::new(), &mut result)?;
return Ok(result);
}
// Find the node corresponding to the prefix
let chars: Vec<char> = prefix.chars().collect();
let node_id = match find_prefix_node(tree, root_id, &chars, 0) {
Ok(id) => id,
Err(_) => return Ok(Vec::new()), // Prefix not found, return empty list
};
// For empty prefix, we start with an empty string
// For non-empty prefix, we start with the prefix minus the last character
// (since the last character is in the node we found)
let prefix_base = if chars.len() > 1 {
chars[0..chars.len() - 1].iter().collect()
} else {
String::new()
};
// Collect all keys from the subtree
collect_keys_with_prefix(tree, node_id, prefix_base, &mut result)?;
Ok(result)
}
/// Finds the node corresponding to a prefix.
fn find_prefix_node(
tree: &mut TST,
node_id: u32,
chars: &[char],
pos: usize,
) -> Result<u32, Error> {
if pos >= chars.len() {
return Ok(node_id);
}
let node = tree.get_node(node_id)?;
let current_char = chars[pos];
if current_char < node.character {
// Go left
if let Some(left_id) = node.left_id {
find_prefix_node(tree, left_id, chars, pos)
} else {
Err(Error::PrefixNotFound(chars.iter().collect()))
}
} else if current_char > node.character {
// Go right
if let Some(right_id) = node.right_id {
find_prefix_node(tree, right_id, chars, pos)
} else {
Err(Error::PrefixNotFound(chars.iter().collect()))
}
} else {
// Character matches
if pos + 1 >= chars.len() {
// This is the last character of the prefix
Ok(node_id)
} else if let Some(middle_id) = node.middle_id {
// Go to next character
find_prefix_node(tree, middle_id, chars, pos + 1)
} else {
Err(Error::PrefixNotFound(chars.iter().collect()))
}
}
}
/// Collects all keys with a given prefix.
fn collect_keys_with_prefix(
tree: &mut TST,
node_id: u32,
current_path: String,
result: &mut Vec<String>,
) -> Result<(), Error> {
let node = tree.get_node(node_id)?;
let mut new_path = current_path.clone();
// For non-root nodes, add the character to the path
if node.character != '\0' {
new_path.push(node.character);
}
// If this node is an end of key, add it to the result
if node.is_end_of_key {
result.push(new_path.clone());
}
// Recursively collect keys from all children
if let Some(left_id) = node.left_id {
collect_keys_with_prefix(tree, left_id, current_path.clone(), result)?;
}
if let Some(middle_id) = node.middle_id {
collect_keys_with_prefix(tree, middle_id, new_path.clone(), result)?;
}
if let Some(right_id) = node.right_id {
collect_keys_with_prefix(tree, right_id, current_path.clone(), result)?;
}
Ok(())
}
/// Recursively collects all keys under a node.
fn collect_all_keys(
tree: &mut TST,
node_id: u32,
current_path: String,
result: &mut Vec<String>,
) -> Result<(), Error> {
let node = tree.get_node(node_id)?;
let mut new_path = current_path.clone();
// Skip adding the character for the root node
if node.character != '\0' {
new_path.push(node.character);
}
// If this node is an end of key, add it to the result
if node.is_end_of_key {
result.push(new_path.clone());
}
// Recursively collect keys from all children
if let Some(left_id) = node.left_id {
collect_all_keys(tree, left_id, current_path.clone(), result)?;
}
if let Some(middle_id) = node.middle_id {
collect_all_keys(tree, middle_id, new_path.clone(), result)?;
}
if let Some(right_id) = node.right_id {
collect_all_keys(tree, right_id, current_path.clone(), result)?;
}
Ok(())
}
/// Gets all values for keys with a given prefix.
pub fn getall(tree: &mut TST, prefix: &str) -> Result<Vec<Vec<u8>>, Error> {
// Get all matching keys
let keys = list(tree, prefix)?;
// Get values for each key
let mut values = Vec::new();
let mut errors = Vec::new();
for key in keys {
match get(tree, &key) {
Ok(value) => values.push(value),
Err(e) => errors.push(format!("Error getting value for key '{}': {:?}", key, e)),
}
}
// If we couldn't get any values but had keys, return the first error
if values.is_empty() && !errors.is_empty() {
return Err(Error::InvalidOperation(errors.join("; ")));
}
Ok(values)
}
impl TST {
/// Helper function to get a node from the database.
pub(crate) fn get_node(&mut self, node_id: u32) -> Result<TSTNode, Error> {
match self.db.get(node_id) {
Ok(data) => TSTNode::deserialize(&data),
Err(err) => Err(Error::OurDB(err)),
}
}
/// Helper function to save a node to the database.
pub(crate) fn save_node(&mut self, node_id: Option<u32>, node: &TSTNode) -> Result<u32, Error> {
let data = node.serialize();
let args = OurDBSetArgs {
id: node_id,
data: &data,
};
match self.db.set(args) {
Ok(id) => Ok(id),
Err(err) => Err(Error::OurDB(err)),
}
}
}

129
tst/src/serialize.rs
View File

@@ -1,129 +0,0 @@
//! Serialization and deserialization for TST nodes.
use crate::error::Error;
use crate::node::TSTNode;
/// Current binary format version.
const VERSION: u8 = 1;
impl TSTNode {
/// Serializes a node to bytes for storage.
pub fn serialize(&self) -> Vec<u8> {
let mut buffer = Vec::new();
// Version
buffer.push(VERSION);
// Character (as UTF-32)
let char_bytes = (self.character as u32).to_le_bytes();
buffer.extend_from_slice(&char_bytes);
// Is end of key
buffer.push(if self.is_end_of_key { 1 } else { 0 });
// Value (only if is_end_of_key)
if self.is_end_of_key {
let value_len = (self.value.len() as u32).to_le_bytes();
buffer.extend_from_slice(&value_len);
buffer.extend_from_slice(&self.value);
} else {
// Zero length
buffer.extend_from_slice(&[0, 0, 0, 0]);
}
// Child pointers
let left_id = self.left_id.unwrap_or(0).to_le_bytes();
buffer.extend_from_slice(&left_id);
let middle_id = self.middle_id.unwrap_or(0).to_le_bytes();
buffer.extend_from_slice(&middle_id);
let right_id = self.right_id.unwrap_or(0).to_le_bytes();
buffer.extend_from_slice(&right_id);
buffer
}
/// Deserializes bytes to a node.
pub fn deserialize(data: &[u8]) -> Result<Self, Error> {
if data.len() < 14 {
// Minimum size: version + char + is_end + value_len + 3 child IDs
return Err(Error::Deserialization("Data too short".to_string()));
}
let mut pos = 0;
// Version
let version = data[pos];
pos += 1;
if version != VERSION {
return Err(Error::Deserialization(format!(
"Unsupported version: {}",
version
)));
}
// Character
let char_bytes = [data[pos], data[pos + 1], data[pos + 2], data[pos + 3]];
let char_code = u32::from_le_bytes(char_bytes);
let character = char::from_u32(char_code)
.ok_or_else(|| Error::Deserialization("Invalid character".to_string()))?;
pos += 4;
// Is end of key
let is_end_of_key = data[pos] != 0;
pos += 1;
// Value length
let value_len_bytes = [data[pos], data[pos + 1], data[pos + 2], data[pos + 3]];
let value_len = u32::from_le_bytes(value_len_bytes) as usize;
pos += 4;
// Value
let value = if value_len > 0 {
if pos + value_len > data.len() {
return Err(Error::Deserialization(
"Value length exceeds data".to_string(),
));
}
data[pos..pos + value_len].to_vec()
} else {
Vec::new()
};
pos += value_len;
// Child pointers
if pos + 12 > data.len() {
return Err(Error::Deserialization(
"Data too short for child pointers".to_string(),
));
}
let left_id_bytes = [data[pos], data[pos + 1], data[pos + 2], data[pos + 3]];
let left_id = u32::from_le_bytes(left_id_bytes);
pos += 4;
let middle_id_bytes = [data[pos], data[pos + 1], data[pos + 2], data[pos + 3]];
let middle_id = u32::from_le_bytes(middle_id_bytes);
pos += 4;
let right_id_bytes = [data[pos], data[pos + 1], data[pos + 2], data[pos + 3]];
let right_id = u32::from_le_bytes(right_id_bytes);
Ok(TSTNode {
character,
value,
is_end_of_key,
left_id: if left_id == 0 { None } else { Some(left_id) },
middle_id: if middle_id == 0 {
None
} else {
Some(middle_id)
},
right_id: if right_id == 0 { None } else { Some(right_id) },
})
}
}
// Function removed as it was unused

294
tst/tests/basic_test.rs
View File

@@ -1,294 +0,0 @@
use std::env::temp_dir;
use std::fs;
use std::time::SystemTime;
use tst::TST;
fn get_test_db_path() -> String {
let timestamp = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_nanos();
let path = temp_dir().join(format!("tst_test_{}", timestamp));
// If the path exists, remove it first
if path.exists() {
let _ = fs::remove_dir_all(&path);
}
// Create the directory
fs::create_dir_all(&path).unwrap();
path.to_string_lossy().to_string()
}
fn cleanup_test_db(path: &str) {
// Make sure to clean up properly
let _ = fs::remove_dir_all(path);
}
#[test]
fn test_create_tst() {
let path = get_test_db_path();
let result = TST::new(&path, true);
match &result {
Ok(_) => (),
Err(e) => println!("Error creating TST: {:?}", e),
}
assert!(result.is_ok());
if let Ok(mut tst) = result {
// Make sure we can perform a basic operation
let set_result = tst.set("test_key", b"test_value".to_vec());
assert!(set_result.is_ok());
}
cleanup_test_db(&path);
}
#[test]
fn test_set_and_get() {
let path = get_test_db_path();
// Create a new TST with reset=true to ensure a clean state
let result = TST::new(&path, true);
assert!(result.is_ok());
let mut tree = result.unwrap();
// Test setting and getting a key
let key = "test_key";
let value = b"test_value".to_vec();
let set_result = tree.set(key, value.clone());
assert!(set_result.is_ok());
let get_result = tree.get(key);
assert!(get_result.is_ok());
assert_eq!(get_result.unwrap(), value);
// Make sure to clean up properly
cleanup_test_db(&path);
}
#[test]
fn test_get_nonexistent_key() {
let path = get_test_db_path();
let mut tree = TST::new(&path, true).unwrap();
// Test getting a key that doesn't exist
let get_result = tree.get("nonexistent_key");
assert!(get_result.is_err());
cleanup_test_db(&path);
}
#[test]
fn test_delete() {
let path = get_test_db_path();
// Create a new TST with reset=true to ensure a clean state
let result = TST::new(&path, true);
assert!(result.is_ok());
let mut tree = result.unwrap();
// Set a key
let key = "delete_test";
let value = b"to_be_deleted".to_vec();
let set_result = tree.set(key, value);
assert!(set_result.is_ok());
// Verify it exists
let get_result = tree.get(key);
assert!(get_result.is_ok());
// Delete it
let delete_result = tree.delete(key);
assert!(delete_result.is_ok());
// Verify it's gone
let get_after_delete = tree.get(key);
assert!(get_after_delete.is_err());
// Make sure to clean up properly
cleanup_test_db(&path);
}
#[test]
fn test_multiple_keys() {
let path = get_test_db_path();
// Create a new TST with reset=true to ensure a clean state
let result = TST::new(&path, true);
assert!(result.is_ok());
let mut tree = result.unwrap();
// Insert multiple keys - use fewer keys to avoid filling the lookup table
let keys = ["apple", "banana", "cherry"];
for (i, key) in keys.iter().enumerate() {
let value = format!("value_{}", i).into_bytes();
let set_result = tree.set(key, value);
// Print error if set fails
if set_result.is_err() {
println!("Error setting key '{}': {:?}", key, set_result);
}
assert!(set_result.is_ok());
}
// Verify all keys exist
for (i, key) in keys.iter().enumerate() {
let expected_value = format!("value_{}", i).into_bytes();
let get_result = tree.get(key);
assert!(get_result.is_ok());
assert_eq!(get_result.unwrap(), expected_value);
}
// Make sure to clean up properly
cleanup_test_db(&path);
}
#[test]
fn test_list_prefix() {
let path = get_test_db_path();
// Create a new TST with reset=true to ensure a clean state
let result = TST::new(&path, true);
assert!(result.is_ok());
let mut tree = result.unwrap();
// Insert keys with common prefixes - use fewer keys to avoid filling the lookup table
let keys = ["apple", "application", "append", "banana", "bandana"];
for key in &keys {
let set_result = tree.set(key, key.as_bytes().to_vec());
assert!(set_result.is_ok());
}
// Test prefix "app"
let list_result = tree.list("app");
assert!(list_result.is_ok());
let app_keys = list_result.unwrap();
// Print the keys for debugging
println!("Keys with prefix 'app':");
for key in &app_keys {
println!(" {}", key);
}
// Check that each key is present
assert!(app_keys.contains(&"apple".to_string()));
assert!(app_keys.contains(&"application".to_string()));
assert!(app_keys.contains(&"append".to_string()));
// Test prefix "ban"
let list_result = tree.list("ban");
assert!(list_result.is_ok());
let ban_keys = list_result.unwrap();
assert!(ban_keys.contains(&"banana".to_string()));
assert!(ban_keys.contains(&"bandana".to_string()));
// Test non-existent prefix
let list_result = tree.list("z");
assert!(list_result.is_ok());
let z_keys = list_result.unwrap();
assert_eq!(z_keys.len(), 0);
// Make sure to clean up properly
cleanup_test_db(&path);
}
#[test]
fn test_getall_prefix() {
let path = get_test_db_path();
// Create a new TST with reset=true to ensure a clean state
let result = TST::new(&path, true);
assert!(result.is_ok());
let mut tree = result.unwrap();
// Insert keys with common prefixes - use fewer keys to avoid filling the lookup table
let keys = ["apple", "application", "append"];
for key in &keys {
let set_result = tree.set(key, key.as_bytes().to_vec());
assert!(set_result.is_ok());
}
// Test getall with prefix "app"
let getall_result = tree.getall("app");
assert!(getall_result.is_ok());
let app_values = getall_result.unwrap();
// Convert values to strings for easier comparison
let app_value_strings: Vec<String> = app_values
.iter()
.map(|v| String::from_utf8_lossy(v).to_string())
.collect();
// Print the values for debugging
println!("Values with prefix 'app':");
for value in &app_value_strings {
println!(" {}", value);
}
// Check that each value is present
assert!(app_value_strings.contains(&"apple".to_string()));
assert!(app_value_strings.contains(&"application".to_string()));
assert!(app_value_strings.contains(&"append".to_string()));
// Make sure to clean up properly
cleanup_test_db(&path);
}
#[test]
fn test_empty_prefix() {
let path = get_test_db_path();
// Create a new TST with reset=true to ensure a clean state
let result = TST::new(&path, true);
assert!(result.is_ok());
let mut tree = result.unwrap();
// Insert some keys
let keys = ["apple", "banana", "cherry"];
for key in &keys {
let set_result = tree.set(key, key.as_bytes().to_vec());
assert!(set_result.is_ok());
}
// Test list with empty prefix (should return all keys)
let list_result = tree.list("");
assert!(list_result.is_ok());
let all_keys = list_result.unwrap();
// Print the keys for debugging
println!("Keys with empty prefix:");
for key in &all_keys {
println!(" {}", key);
}
// Check that each key is present
for key in &keys {
assert!(all_keys.contains(&key.to_string()));
}
// Make sure to clean up properly
cleanup_test_db(&path);
}

267
tst/tests/prefix_test.rs
View File

@@ -1,267 +0,0 @@
use std::env::temp_dir;
use std::fs;
use std::time::SystemTime;
use tst::TST;
fn get_test_db_path() -> String {
let timestamp = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap()
.as_nanos();
let path = temp_dir().join(format!("tst_prefix_test_{}", timestamp));
// If the path exists, remove it first
if path.exists() {
let _ = fs::remove_dir_all(&path);
}
// Create the directory
fs::create_dir_all(&path).unwrap();
path.to_string_lossy().to_string()
}
fn cleanup_test_db(path: &str) {
// Make sure to clean up properly
let _ = fs::remove_dir_all(path);
}
#[test]
fn test_prefix_with_common_prefixes() {
let path = get_test_db_path();
let mut tree = TST::new(&path, true).unwrap();
// Insert keys with common prefixes
let test_data = [
("test", b"value1".to_vec()),
("testing", b"value2".to_vec()),
("tested", b"value3".to_vec()),
("tests", b"value4".to_vec()),
("tester", b"value5".to_vec()),
];
for (key, value) in &test_data {
tree.set(key, value.clone()).unwrap();
}
// Test prefix "test"
let keys = tree.list("test").unwrap();
assert_eq!(keys.len(), 5);
for (key, _) in &test_data {
assert!(keys.contains(&key.to_string()));
}
// Test prefix "teste"
let keys = tree.list("teste").unwrap();
assert_eq!(keys.len(), 2);
assert!(keys.contains(&"tested".to_string()));
assert!(keys.contains(&"tester".to_string()));
cleanup_test_db(&path);
}
#[test]
fn test_prefix_with_different_prefixes() {
let path = get_test_db_path();
let mut tree = TST::new(&path, true).unwrap();
// Insert keys with different prefixes
let test_data = [
("apple", b"fruit1".to_vec()),
("banana", b"fruit2".to_vec()),
("cherry", b"fruit3".to_vec()),
("date", b"fruit4".to_vec()),
("elderberry", b"fruit5".to_vec()),
];
for (key, value) in &test_data {
tree.set(key, value.clone()).unwrap();
}
// Test each prefix
for (key, _) in &test_data {
let prefix = &key[0..1]; // First character
let keys = tree.list(prefix).unwrap();
assert!(keys.contains(&key.to_string()));
}
// Test non-existent prefix
let keys = tree.list("z").unwrap();
assert_eq!(keys.len(), 0);
cleanup_test_db(&path);
}
#[test]
fn test_prefix_with_empty_string() {
let path = get_test_db_path();
// Create a new TST with reset=true to ensure a clean state
let result = TST::new(&path, true);
assert!(result.is_ok());
let mut tree = result.unwrap();
// Insert some keys
let test_data = [
("apple", b"fruit1".to_vec()),
("banana", b"fruit2".to_vec()),
("cherry", b"fruit3".to_vec()),
];
for (key, value) in &test_data {
let set_result = tree.set(key, value.clone());
assert!(set_result.is_ok());
}
// Test empty prefix (should return all keys)
let list_result = tree.list("");
assert!(list_result.is_ok());
let keys = list_result.unwrap();
// Print the keys for debugging
println!("Keys with empty prefix:");
for key in &keys {
println!(" {}", key);
}
// Check that each key is present
for (key, _) in &test_data {
assert!(keys.contains(&key.to_string()));
}
// Make sure to clean up properly
cleanup_test_db(&path);
}
#[test]
fn test_getall_with_prefix() {
let path = get_test_db_path();
let mut tree = TST::new(&path, true).unwrap();
// Insert keys with common prefixes
let test_data = [
("test", b"value1".to_vec()),
("testing", b"value2".to_vec()),
("tested", b"value3".to_vec()),
("tests", b"value4".to_vec()),
("tester", b"value5".to_vec()),
];
for (key, value) in &test_data {
tree.set(key, value.clone()).unwrap();
}
// Test getall with prefix "test"
let values = tree.getall("test").unwrap();
assert_eq!(values.len(), 5);
for (_, value) in &test_data {
assert!(values.contains(value));
}
cleanup_test_db(&path);
}
#[test]
fn test_prefix_with_unicode_characters() {
let path = get_test_db_path();
let mut tree = TST::new(&path, true).unwrap();
// Insert keys with Unicode characters
let test_data = [
("café", b"coffee".to_vec()),
("cafétéria", b"cafeteria".to_vec()),
("caffè", b"italian coffee".to_vec()),
("café au lait", b"coffee with milk".to_vec()),
];
for (key, value) in &test_data {
tree.set(key, value.clone()).unwrap();
}
// Test prefix "café"
let keys = tree.list("café").unwrap();
// Print the keys for debugging
println!("Keys with prefix 'café':");
for key in &keys {
println!(" {}", key);
}
// Check that the keys we expect are present
assert!(keys.contains(&"café".to_string()));
assert!(keys.contains(&"café au lait".to_string()));
// We don't assert on the exact count because Unicode handling can vary
// Test prefix "caf"
let keys = tree.list("caf").unwrap();
// Print the keys for debugging
println!("Keys with prefix 'caf':");
for key in &keys {
println!(" {}", key);
}
// Check that each key is present individually
// Due to Unicode handling, we need to be careful with exact matching
// The important thing is that we can find the keys we need
// Check that we have at least the café and café au lait keys
assert!(keys.contains(&"café".to_string()));
assert!(keys.contains(&"café au lait".to_string()));
// We don't assert on the exact count because Unicode handling can vary
cleanup_test_db(&path);
}
#[test]
fn test_prefix_with_long_keys() {
let path = get_test_db_path();
let mut tree = TST::new(&path, true).unwrap();
// Insert long keys
let test_data = [
(
"this_is_a_very_long_key_for_testing_purposes_1",
b"value1".to_vec(),
),
(
"this_is_a_very_long_key_for_testing_purposes_2",
b"value2".to_vec(),
),
(
"this_is_a_very_long_key_for_testing_purposes_3",
b"value3".to_vec(),
),
("this_is_another_long_key_for_testing", b"value4".to_vec()),
];
for (key, value) in &test_data {
tree.set(key, value.clone()).unwrap();
}
// Test prefix "this_is_a_very"
let keys = tree.list("this_is_a_very").unwrap();
assert_eq!(keys.len(), 3);
// Test prefix "this_is"
let keys = tree.list("this_is").unwrap();
assert_eq!(keys.len(), 4);
for (key, _) in &test_data {
assert!(keys.contains(&key.to_string()));
}
cleanup_test_db(&path);
}