diff --git a/rebar.config b/rebar.config index 03ca19b31..e175ed887 100644 --- a/rebar.config +++ b/rebar.config @@ -117,7 +117,7 @@ ]}. {deps, [ - {elmdb, { git, "https://github.com/twilson63/elmdb-rs.git", {branch, "feat/match" }}}, + {elmdb, { git, "https://github.com/twilson63/elmdb-rs.git", {ref, "5255868638e91b4dff24163467765d780f8a6f4a" }}}, {b64fast, {git, "https://github.com/ArweaveTeam/b64fast.git", {ref, "58f0502e49bf73b29d95c6d02460d1fb8d2a5273"}}}, {cowlib, "2.16.0"}, {cowboy, "2.14.0"}, diff --git a/rebar.lock b/rebar.lock index abe0deb60..2be4a7dbc 100644 --- a/rebar.lock +++ b/rebar.lock @@ -9,7 +9,7 @@ {<<"ddskerl">>,{pkg,<<"ddskerl">>,<<"0.4.2">>},1}, {<<"elmdb">>, {git,"https://github.com/twilson63/elmdb-rs.git", - {ref,"90c8857cd4ccff341fbe415b96bc5703d17ff7f0"}}, + {ref,"5255868638e91b4dff24163467765d780f8a6f4a"}}, 0}, {<<"graphql">>,{pkg,<<"graphql_erl">>,<<"0.17.1">>},0}, {<<"gun">>,{pkg,<<"gun">>,<<"2.2.0">>},0}, diff --git a/src/hb_app.erl b/src/hb_app.erl index 83dcb1028..f66645943 100644 --- a/src/hb_app.erl +++ b/src/hb_app.erl @@ -7,7 +7,7 @@ -behaviour(application). --export([start/2, stop/1]). +-export([start/2, prep_stop/1, stop/1]). -include("include/hb.hrl"). @@ -18,5 +18,24 @@ start(_StartType, _StartArgs) -> _TimestampServer = ar_timestamp:start(), {ok, _} = hb_http_server:start(). +prep_stop(State) -> + maybe + {ok, Opts} ?= find_lmdb_store(), + hb_store_lmdb:flush(Opts) + end, + State. + stop(_State) -> - ok. \ No newline at end of file + maybe + {ok, Opts} ?= find_lmdb_store(), + hb_store_lmdb:stop(Opts) + end, + ok. + +find_lmdb_store() -> + Stores = maps:get(store, hb_opts:default_message()), + Pred = fun(S) -> maps:get(<<"store-module">>, S) == hb_store_lmdb end, + case lists:search(Pred, Stores) of + {value, Opts} -> {ok, Opts}; + false -> not_found + end. \ No newline at end of file diff --git a/src/hb_opts.erl b/src/hb_opts.erl index 8d239d5a8..63f4b58d5 100644 --- a/src/hb_opts.erl +++ b/src/hb_opts.erl @@ -299,7 +299,7 @@ default_message() -> ?DEFAULT_PRIMARY_STORE, #{ <<"store-module">> => hb_store_fs, - <<"name">> => <<"cache-mainnet">> + <<"name">> => <<"cache-mainnet/fs">> }, #{ <<"store-module">> => hb_store_gateway, diff --git a/src/hb_store.erl b/src/hb_store.erl index 89c5daa3d..7fef14fa3 100644 --- a/src/hb_store.erl +++ b/src/hb_store.erl @@ -46,7 +46,7 @@ -export([behavior_info/1]). -export([start/1, stop/1, reset/1]). -export([filter/2, scope/2, sort/2]). --export([type/2, read/2, write/3, list/2, match/2]). +-export([type/2, read/2, write/3, sync/2, list/2, match/2]). -export([path/1, path/2, add_path/2, add_path/3, join/1]). -export([make_group/2, make_link/3, resolve/2]). -export([find/1]). @@ -305,6 +305,71 @@ list(Modules, Path) -> call_function(Modules, list, [Path]). %% is given as a list of IDs. match(Modules, Match) -> call_function(Modules, match, [Match]). +%% @doc Copies the contents of one store to another. +sync(#{<<"store-module">> := hb_store_lmdb} = FromStore, ToStore) -> + ?event({sync_start, FromStore, ToStore}), + FromStoreOpts = maps:put(<<"resolve">>, false, FromStore), + Res = hb_store_lmdb:fold_while(FromStoreOpts, fun({Key, Value}, {ok, Acc}) -> + case hb_store:write(ToStore, Key, Value) of + ok -> + {cont, {ok, Acc + 1}}; + Error -> + ?event({sync_error, Error}), + {halt, {error, sync_failed}} + end + end, {ok, 0}), + maybe + {ok, Count} ?= Res, + ?event({sync_success, Count}), + ok + end; + +sync(#{<<"store-module">> := hb_store_fs} = FromStore, ToStore) -> + ?event({sync_start, FromStore, ToStore}), + FromStoreOpts = maps:put(<<"resolve">>, false, FromStore), + maybe + {ok, Entries} ?= hb_store:list(FromStore, <<"/">>), + case sync_fs_entries(Entries, <<"/">>, FromStoreOpts, ToStore) of + [] -> ok; + FailedKeyValues -> {error, {sync_failed, FailedKeyValues}} + end + end. + +sync_fs_entries(Entries, ParentDir, FromStore, ToStore) -> + ?event({sync_entries, ParentDir, Entries}), + lists:foldl(fun(Key, Acc) -> + NewPath = + case ParentDir of + Bin when Bin == <<"">> orelse Bin == <<"/">> -> Key; + _ -> <> + end, + case type(FromStore, NewPath) of + Type when Type == simple orelse Type == link -> + case hb_store:read(FromStore, NewPath) of + {ok, Value} when Type == simple -> + case hb_store:write(ToStore, NewPath, Value) of + ok -> Acc; + _Error -> [{NewPath, Value} | Acc] + end; + {ok, LinkTarget} when Type == link -> + ok = hb_store:make_link(ToStore, LinkTarget, NewPath), + Acc; + _Error -> + [{NewPath, undefined} | Acc] + end; + composite -> + case hb_store:make_group(ToStore, NewPath) of + ok -> + {ok, Entries2} = hb_store:list(FromStore, NewPath), + Acc ++ sync_fs_entries(Entries2, NewPath, FromStore, ToStore); + _Error -> + [{NewPath, undefined} | Acc] + end; + not_found -> + Acc + end + end, [], Entries). + %% @doc Call a function on the first store module that succeeds. Returns its %% result, or `not_found` if none of the stores succeed. If `TIME_CALLS` is set, %% this function will also time the call and increment the appropriate event @@ -411,7 +476,7 @@ call_all(X, _Function, _Args) when not is_list(X) -> call_all([], _Function, _Args) -> ok; call_all([Store = #{<<"store-module">> := Mod} | Rest], Function, Args) -> - try apply_store_function(Mod, Function, Store, Args) + try apply_store_function(Mod, Store, Function, Args) catch Class:Reason:Stacktrace -> ?event(warning, {store_call_failed, {Class, Reason, Stacktrace}}), @@ -513,11 +578,144 @@ hierarchical_path_resolution_test(Store) -> hb_store:read(Store, [<<"test-link">>, <<"test-file">>]) ). +%% @doc Test the hb_store:sync function by syncing from hb_store_fs to hb_store_lmdb +hb_store_sync_test(_Store) -> + % Generate unique names to avoid conflicts + TestId = integer_to_binary(erlang:system_time(microsecond)), + % Set up FromStore (hb_store_fs) with resolve=false as specified + FromStore = #{ + <<"store-module">> => hb_store_fs, + <<"name">> => <<"cache-sync-from-", TestId/binary>>, + <<"resolve">> => false + }, + % Set up ToStore (hb_store_lmdb) + ToStore = #{ + <<"store-module">> => hb_store_lmdb, + <<"name">> => <<"cache-sync-to-", TestId/binary>> + }, + + % Clean up any existing data + hb_store:reset(FromStore), + hb_store:reset(ToStore), + + % Start both stores + hb_store:start(FromStore), + hb_store:start(ToStore), + + % Populate FromStore with directories, files, and links + % Create a directory structure + ok = hb_store:make_group(FromStore, <<"test-dir">>), + ok = hb_store:write(FromStore, <<"test-dir/file1.txt">>, <<"Hello World">>), + ok = hb_store:write(FromStore, <<"test-dir/file2.txt">>, <<"Test Data">>), + + % Create a nested directory + ok = hb_store:make_group(FromStore, <<"test-dir/nested">>), + ok = hb_store:write(FromStore, <<"test-dir/nested/deep-file.txt">>, <<"Deep Content">>), + + % Create some top-level files + ok = hb_store:write(FromStore, <<"root-file.txt">>, <<"Root Content">>), + + % Create a link + ok = hb_store:make_link(FromStore, <<"root-file.txt">>, <<"link-to-root">>), + + % Perform the sync operation + Result = hb_store:sync(FromStore, ToStore), + ?assertEqual(ok, Result), + + % Verify that directories exist in ToStore + ?assertEqual(composite, hb_store:type(ToStore, <<"test-dir">>)), + ?assertEqual(composite, hb_store:type(ToStore, <<"test-dir/nested">>)), + + % Verify that files exist in ToStore + {ok, File1Content} = hb_store:read(ToStore, <<"test-dir/file1.txt">>), + ?assertEqual(<<"Hello World">>, File1Content), + + {ok, File2Content} = hb_store:read(ToStore, <<"test-dir/file2.txt">>), + ?assertEqual(<<"Test Data">>, File2Content), + + {ok, DeepContent} = hb_store:read(ToStore, <<"test-dir/nested/deep-file.txt">>), + ?assertEqual(<<"Deep Content">>, DeepContent), + + {ok, RootContent} = hb_store:read(ToStore, <<"root-file.txt">>), + ?assertEqual(<<"Root Content">>, RootContent), + + % Verify that links work in ToStore + {ok, LinkContent} = hb_store:read(ToStore, <<"link-to-root">>), + ?assertEqual(<<"Root Content">>, LinkContent), + + % Clean up + hb_store:stop(FromStore), + hb_store:stop(ToStore). + +%% @doc Test the hb_store:sync function by syncing from hb_store_lmdb to hb_store_lmdb +hb_store_lmdb_sync_test(_Store) -> + % Generate unique names to avoid conflicts + TestId = integer_to_binary(erlang:system_time(microsecond)), + % Set up FromStore (hb_store_lmdb) with resolve=false as specified + FromStore = #{ + <<"store-module">> => hb_store_lmdb, + <<"name">> => <<"cache-lmdb-sync-from-", TestId/binary>>, + <<"resolve">> => false + }, + % Set up ToStore (hb_store_lmdb) + ToStore = #{ + <<"store-module">> => hb_store_lmdb, + <<"name">> => <<"cache-lmdb-sync-to-", TestId/binary>> + }, + + % Clean up any existing data + hb_store:reset(FromStore), + hb_store:reset(ToStore), + + % Start both stores + hb_store:start(FromStore), + hb_store:start(ToStore), + + % Populate FromStore with data + ok = hb_store:write(FromStore, <<"key1">>, <<"value1">>), + ok = hb_store:write(FromStore, <<"key2">>, <<"value2">>), + ok = hb_store:write(FromStore, <<"nested/key3">>, <<"value3">>), + ok = hb_store:write(FromStore, <<"deep/nested/key4">>, <<"value4">>), + + % Create some links + ok = hb_store:make_link(FromStore, <<"key1">>, <<"link-to-key1">>), + ok = hb_store:make_link(FromStore, <<"nested/key3">>, <<"link-to-nested">>), + + % Perform the sync operation + Result = hb_store:sync(FromStore, ToStore), + ?assertEqual(ok, Result), + + % Verify that all data exists in ToStore + {ok, Value1} = hb_store:read(ToStore, <<"key1">>), + ?assertEqual(<<"value1">>, Value1), + + {ok, Value2} = hb_store:read(ToStore, <<"key2">>), + ?assertEqual(<<"value2">>, Value2), + + {ok, Value3} = hb_store:read(ToStore, <<"nested/key3">>), + ?assertEqual(<<"value3">>, Value3), + + {ok, Value4} = hb_store:read(ToStore, <<"deep/nested/key4">>), + ?assertEqual(<<"value4">>, Value4), + + % Verify that links work in ToStore + {ok, LinkValue1} = hb_store:read(ToStore, <<"link-to-key1">>), + ?assertEqual(<<"value1">>, LinkValue1), + + {ok, LinkValue3} = hb_store:read(ToStore, <<"link-to-nested">>), + ?assertEqual(<<"value3">>, LinkValue3), + + % Clean up + hb_store:stop(FromStore), + hb_store:stop(ToStore). + store_suite_test_() -> generate_test_suite([ {"simple path resolution", fun simple_path_resolution_test/1}, {"resursive path resolution", fun resursive_path_resolution_test/1}, - {"hierarchical path resolution", fun hierarchical_path_resolution_test/1} + {"hierarchical path resolution", fun hierarchical_path_resolution_test/1}, + {"hb_store sync", fun hb_store_sync_test/1}, + {"hb_store lmdb sync", fun hb_store_lmdb_sync_test/1} ]). benchmark_suite_test_() -> diff --git a/src/hb_store_fs.erl b/src/hb_store_fs.erl index ac2bf8849..72e1c2ab8 100644 --- a/src/hb_store_fs.erl +++ b/src/hb_store_fs.erl @@ -10,7 +10,7 @@ %%% requires (many reads and writes to explicit keys, few directory 'listing' or %%% search operations), awhile others perform suboptimally. %%% -%%% Additionally, thisstore implementation offers the ability for simple +%%% Additionally, this store implementation offers the ability for simple %%% integration of HyperBEAM with other non-volatile storage media: `hb_store_fs' %%% will interact with any service that implements the host operating system's %%% native filesystem API. By mounting devices via `FUSE' (etc), HyperBEAM is @@ -45,22 +45,33 @@ reset(#{ <<"name">> := DataDir }) -> ?event({reset_store, {path, DataDir}}). %% @doc Read a key from the store, following symlinks as needed. +read(#{<<"resolve">> := false} = Opts, Key) -> + maybe {prefixed_link, LinkTarget} ?= read_path(add_prefix(Opts, Key), false), + case remove_prefix(Opts, LinkTarget) of + <<"/", Path/binary>> -> {ok, Path}; + Path -> {ok, Path} + end + end; read(Opts, Key) -> - read(add_prefix(Opts, resolve(Opts, Key))). -read(Path) -> - ?event({read, Path}), - case file:read_file_info(Path) of - {ok, #file_info{type = regular}} -> - {ok, _} = file:read_file(Path); - _ -> - case file:read_link(Path) of - {ok, Link} -> - ?event({link_found, Path, Link}), - read(Link); - _ -> - not_found - end - end. + read_path(add_prefix(Opts, resolve(Opts, Key)), true). + +read_path(Path, FollowLink) -> + ?event({read, Path}), + case file:read_file_info(Path) of + {ok, #file_info{type = regular}} -> + {ok, _} = file:read_file(Path); + _ -> + case file:read_link(Path) of + {ok, Link} when FollowLink -> + ?event({link_found, Path, Link}), + read_path(Link, true); + {ok, LinkTarget} -> + ?event({link_found_ret, Path, LinkTarget}), + {prefixed_link, list_to_binary(LinkTarget)}; + _ -> + not_found + end + end. %% @doc Write a value to the specified path in the store. write(Opts, PathComponents, Value) -> @@ -113,16 +124,20 @@ resolve(Opts, CurrPath, [Next|Rest]) -> %% @doc Determine the type of a key in the store. type(Opts, Key) -> - type(add_prefix(Opts, Key)). -type(Path) -> + FollowLink = maps:get(<<"resolve">>, Opts, true), + type_path(add_prefix(Opts, Key), FollowLink). + +type_path(Path, FollowLink) -> ?event({type, Path}), case file:read_file_info(Path) of {ok, #file_info{type = directory}} -> composite; {ok, #file_info{type = regular}} -> simple; _ -> case file:read_link(Path) of - {ok, Link} -> - type(Link); + {ok, Link} when FollowLink -> + type_path(Link, true); + {ok, _Link} -> + link; _ -> not_found end @@ -135,7 +150,7 @@ make_group(Opts = #{ <<"name">> := _DataDir }, Path) -> % We need to ensure that the parent directory exists, so that we can % make the group. filelib:ensure_dir(P), - case file:make_dir(P) of + case file:make_dir(P) of ok -> ok; {error, eexist} -> ok end. @@ -144,8 +159,8 @@ make_group(Opts = #{ <<"name">> := _DataDir }, Path) -> make_link(_, Link, Link) -> ok; make_link(Opts, Existing, New) -> ?event({symlink, - add_prefix(Opts, Existing), - P2 = add_prefix(Opts, New)}), + add_prefix(Opts, Existing), + P2 = add_prefix(Opts, New)}), filelib:ensure_dir(P2), case file:make_symlink(add_prefix(Opts, Existing), N = add_prefix(Opts, New)) of ok -> ok; @@ -164,7 +179,7 @@ make_link(Opts, Existing, New) -> %% @doc Add the directory prefix to a path. add_prefix(#{ <<"name">> := Prefix }, Path) -> - ?event({add_prefix, Prefix, Path}), + ?event({add_prefix, Prefix, Path}), % Check if the prefix is an absolute path IsAbsolute = is_binary(Prefix) andalso binary:first(Prefix) =:= $/ orelse is_list(Prefix) andalso hd(Prefix) =:= $/, diff --git a/src/hb_store_lmdb.erl b/src/hb_store_lmdb.erl index dd3101bed..1e2d987f8 100644 --- a/src/hb_store_lmdb.erl +++ b/src/hb_store_lmdb.erl @@ -21,7 +21,7 @@ %% Public API exports -export([start/1, stop/1, scope/0, scope/1, reset/1]). --export([read/2, write/3, list/2, match/2]). +-export([read/2, write/3, flush/1, list/2, match/2, fold_while/3]). -export([make_group/2, make_link/3, type/2]). -export([path/2, add_path/3, resolve/2]). @@ -36,7 +36,7 @@ -define(DEFAULT_MAX_FLUSH_TIME, 50). % Maximum time between flushes -define(MAX_REDIRECTS, 1000). % Only resolve 1000 links to data -define(MAX_PENDING_WRITES, 400). % Force flush after x pending --define(FOLD_YIELD_INTERVAL, 100). % Yield every x keys +-define(FOLD_CHUNK_SIZE, 100). % Iterator chunk size for folding %% @doc Start the LMDB storage system for a given database configuration. %% @@ -55,13 +55,18 @@ start(Opts = #{ <<"name">> := DataDir }) -> % Ensure the directory exists before opening LMDB environment DataDirPath = hb_util:list(DataDir), ok = filelib:ensure_dir(filename:join(DataDirPath, "dummy")), + NoSyncParam = + case maps:get(<<"no-sync">>, Opts, true) of + true -> [no_sync]; + false -> [] + end, % Create the LMDB environment with specified size limit {ok, Env} = elmdb:env_open( DataDirPath, [ {map_size, maps:get(<<"capacity">>, Opts, ?DEFAULT_SIZE)}, - no_mem_init, no_sync + no_mem_init | NoSyncParam ] ), {ok, DBInstance} = elmdb:db_open(Env, [create]), @@ -85,14 +90,17 @@ start(_) -> %% @param Opts Database configuration map %% @param Key The key to examine %% @returns 'composite' for group entries, 'simple' for regular values --spec type(map(), binary()) -> composite | simple | not_found. +-spec type(map(), binary()) -> composite | simple | link | not_found. type(Opts, Key) -> + FollowLink = maps:get(<<"resolve">>, Opts, true), case read_direct(Opts, Key) of {ok, Value} -> case is_link(Value) of - {true, Link} -> + {true, Link} when FollowLink -> % This is a link, check the target's type type(Opts, Link); + {true, _Link} -> + link; false -> case Value of <<"group">> -> @@ -141,6 +149,22 @@ write(Opts, Path, Value) -> retry end. +-spec flush(map()) -> ok | {error, flush_failed}. +flush(Opts) -> + #{ <<"env">> := DBEnv } = find_env(Opts), + case elmdb:env_sync(DBEnv) of + ok -> ok; + {error, Type, Description} -> + ?event( + error, + {lmdb_error, + {type, Type}, + {description, Description} + } + ), + {error, flush_failed} + end. + %% @doc Read a value from the database by key, with automatic link resolution. %% %% This function attempts to read a value directly from the committed database. @@ -166,8 +190,11 @@ write(Opts, Path, Value) -> -spec read(map(), binary() | list()) -> {ok, binary()} | {error, term()}. read(Opts, PathParts) when is_list(PathParts) -> read(Opts, to_path(PathParts)); +read(#{<<"resolve">> := false} = Opts, Path) -> + maybe {ok, <<"link:", RawValue/binary>>} ?= read_direct(Opts, Path), + {ok, RawValue} + end; read(Opts, Path) -> - % Try direct read first (fast path for non-link paths) case read_with_links(Opts, Path) of {ok, Value} -> {ok, Value}; @@ -271,7 +298,7 @@ resolve_path_links(Opts, Path, Depth) -> resolve_path_links_acc(_Opts, [], AccPath, _Depth) -> % No more segments to process {ok, lists:reverse(AccPath)}; -resolve_path_links_acc(_, FullPath = [<<"data">>|_], [], _Depth) -> +resolve_path_links_acc(_, FullPath = [<<"data">> | _], [], _Depth) -> {ok, FullPath}; resolve_path_links_acc(Opts, [Head | Tail], AccPath, Depth) -> % Build the accumulated path so far @@ -367,7 +394,6 @@ list(Opts, Path) -> #{ <<"db">> := DBInstance } = find_env(Opts), case elmdb:list(DBInstance, SearchPath) of {ok, Children} -> {ok, Children}; - {error, not_found} -> {ok, []}; % Normalize new error format not_found -> {ok, []} % Handle both old and new format end. @@ -391,10 +417,45 @@ match(Opts, MatchKVs) -> {ok, Matches} -> ?event({elmdb_matched, Matches}), {ok, Matches}; - {error, not_found} -> not_found; not_found -> not_found end. +-spec fold_while(Opts :: map(), Fun :: fun(), Acc0 :: term()) -> + {ok, term()} | not_found | {error, term(), binary()}. +fold_while(Opts, Fun, Acc0) -> + #{ <<"db">> := DBInstance } = find_env(Opts), + maybe + ok ?= elmdb:flush(DBInstance), + {ok, IterRes} ?= elmdb:iterate_start(DBInstance, <<>>, ?FOLD_CHUNK_SIZE), + % Links = [{Key, Value} || {Key, Value} <- KVList, type(FromStore, Key) == link], + % ?debug_print({links, Links}), + fold_chunks(DBInstance, Fun, Acc0, IterRes) + end. + +fold_chunks(DBInstance, Fun, Acc0, {KVList, Continuation}) -> + case do_fold_while(Fun, Acc0, KVList) of + {cont, Acc1} -> + fold_continue(DBInstance, Fun, Acc1, Continuation); + {halt, Res} -> + Res + end. + +do_fold_while(Fun, AccIn, [{Key, Value}]) -> + Fun({Key, Value}, AccIn); + +do_fold_while(Fun, AccIn, [KV | KVList]) -> + maybe + {cont, AccOut} ?= Fun(KV, AccIn), + do_fold_while(Fun, AccOut, KVList) + end. + +fold_continue(_DBInstance, _Fun, Acc1, not_found) -> + Acc1; +fold_continue(DBInstance, Fun, Acc1, Continuation) -> + maybe + {ok, IterRes} ?= elmdb:iterate_cont(DBInstance, Continuation, ?FOLD_CHUNK_SIZE), + fold_chunks(DBInstance, Fun, Acc1, IterRes) + end. %% @doc Create a group entry that can contain other keys hierarchically. %% @@ -413,9 +474,11 @@ match(Opts, MatchKVs) -> %% @param GroupName Binary name for the group %% @returns Result of the write operation -spec make_group(map(), binary()) -> ok | {error, term()}. +make_group(Opts, <<"/", GroupName/binary>>) when is_map(Opts) -> + make_group(Opts, GroupName); make_group(Opts, GroupName) when is_map(Opts), is_binary(GroupName) -> write(Opts, GroupName, <<"group">>); -make_group(_,_) -> +make_group(_, _) -> {error, {badarg, <<"StoreOps must be map and GroupName must be a binary">>}}. %% @doc Ensure all parent groups exist for a given path. @@ -481,10 +544,12 @@ make_link(Opts, Existing, New) when is_list(Existing) -> ExistingBin = to_path(Existing), make_link(Opts, ExistingBin, New); make_link(Opts, Existing, New) -> - ExistingBin = hb_util:bin(Existing), - % Ensure parent groups exist for the new link path (like filesystem ensure_dir) - ensure_parent_groups(Opts, New), - write(Opts, New, <<"link:", ExistingBin/binary>>). + % Ensure key-value is binary pair + ExistingBin = hb_util:bin(Existing), + NewBin = hb_util:bin(New), + % Ensure parent groups exist for the new link path (like filesystem ensure_dir) + ensure_parent_groups(Opts, NewBin), + write(Opts, NewBin, <<"link:", ExistingBin/binary>>). %% @doc Transform a path into the store's canonical form. %% For LMDB, paths are simply joined with "/" separators. @@ -530,7 +595,7 @@ resolve(Opts, PathParts) when is_list(PathParts) -> % If resolution fails, return original path as binary to_path(PathParts) end; -resolve(_,_) -> not_found. +resolve(_, _) -> not_found. %% @doc Retrieve or create the LMDB environment handle for a database. find_env(Opts) -> hb_store:find(Opts). @@ -763,6 +828,26 @@ type_test() -> Type2 = type(StoreOpts, <<"assets/1">>), ?event({type2, Type2}), ?assertEqual(simple, Type2). + +type_link_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/store-7">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + ?event(debug, step1_make_group), + make_group(StoreOpts, <<"test-dir1">>), + ?event(debug, step2_write_file), + write(StoreOpts, [<<"test-dir1">>, <<"test-file">>], <<"test-data">>), + ?event(debug, step3_make_link), + make_link(StoreOpts, [<<"test-dir1">>], <<"test-link">>), + Type1 = type(StoreOpts, <<"test-link">>), + ?assertEqual(composite, Type1), + StoreOpts2 = maps:put(<<"resolve">>, false, StoreOpts), + Type2 = type(StoreOpts2, <<"test-link">>), + ?assertEqual(link, Type2). + %% @doc Link key list test - verifies symbolic link creation using structured key paths. %% @@ -1083,4 +1168,37 @@ list_with_link_test() -> {ok, LinkChildren} = list(StoreOpts, <<"link-to-group">>), ?event({link_children, LinkChildren}), ?assertEqual(ExpectedChildren, lists:sort(LinkChildren)), - stop(StoreOpts). \ No newline at end of file + stop(StoreOpts). + + +%% @doc Read follow test - verifies shallow reads and reads with links. +read_follow_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/lmdb-read-follow">> + }, + reset(StoreOpts), + write(StoreOpts, <<"Hello">>, <<"World">>), + make_link(StoreOpts, <<"Hello">>, <<"HelloLink">>), + {ok, Value} = read(StoreOpts, <<"Hello">>), + ?assertEqual(Value, <<"World">>), + {ok, Value2} = read(StoreOpts, <<"HelloLink">>), + ?assertEqual(Value2, <<"World">>), + StoreOpts2 = maps:put(<<"resolve">>, false, StoreOpts), + {ok, Value3} = read(StoreOpts2, <<"HelloLink">>), + ?assertEqual(Value3, <<"Hello">>), + ok = stop(StoreOpts). +%% @doc Test that list function resolves links correctly +flush_test() -> + StoreOpts = #{ + <<"store-module">> => ?MODULE, + <<"name">> => <<"/tmp/flush">>, + <<"capacity">> => ?DEFAULT_SIZE + }, + reset(StoreOpts), + write(StoreOpts, <<"key1">>, <<"value1">>), + write(StoreOpts, <<"key2">>, <<"value2">>), + write(StoreOpts, <<"key3">>, <<"value3">>), + ?assertEqual(ok, flush(StoreOpts)), + ?assertEqual({ok, <<"value1">>}, read(StoreOpts, <<"key1">>)), + stop(StoreOpts).