feat: sync from an hb store to another (FOR-139)

rebar.config

@@ -117,7 +117,7 @@
 ]}.
 
 {deps, [
-    {elmdb, { git, "https://github.com/twilson63/elmdb-rs.git", {branch, "feat/match" }}},
+    {elmdb, { git, "https://github.com/twilson63/elmdb-rs.git", {ref, "5255868638e91b4dff24163467765d780f8a6f4a" }}},
 	{b64fast, {git, "https://github.com/ArweaveTeam/b64fast.git", {ref, "58f0502e49bf73b29d95c6d02460d1fb8d2a5273"}}},
     {cowlib, "2.16.0"},
 	{cowboy, "2.14.0"},

rebar.lock

@@ -9,7 +9,7 @@
  {<<"ddskerl">>,{pkg,<<"ddskerl">>,<<"0.4.2">>},1},
  {<<"elmdb">>,
   {git,"https://github.com/twilson63/elmdb-rs.git",
-       {ref,"90c8857cd4ccff341fbe415b96bc5703d17ff7f0"}},
+       {ref,"5255868638e91b4dff24163467765d780f8a6f4a"}},
   0},
  {<<"graphql">>,{pkg,<<"graphql_erl">>,<<"0.17.1">>},0},
  {<<"gun">>,{pkg,<<"gun">>,<<"2.2.0">>},0},

src/hb_app.erl

@@ -7,7 +7,7 @@
 
 -behaviour(application).
 
--export([start/2, stop/1]).
+-export([start/2, prep_stop/1, stop/1]).
 
 -include("include/hb.hrl").
 
@@ -18,5 +18,24 @@ start(_StartType, _StartArgs) ->
     _TimestampServer = ar_timestamp:start(),
     {ok, _} = hb_http_server:start().
 
+prep_stop(State) ->
+    maybe
+        {ok, Opts} ?= find_lmdb_store(),
+        hb_store_lmdb:flush(Opts) 
+    end,
+    State.
+
 stop(_State) ->
-    ok.
+    maybe
+        {ok, Opts} ?= find_lmdb_store(),
+        hb_store_lmdb:stop(Opts) 
+    end,
+    ok.
+
+find_lmdb_store() ->
+    Stores = maps:get(store, hb_opts:default_message()),
+    Pred = fun(S) -> maps:get(<<"store-module">>, S) == hb_store_lmdb end,
+    case lists:search(Pred, Stores) of
+        {value, Opts} -> {ok, Opts};
+        false -> not_found
+    end.

src/hb_opts.erl

@@ -299,7 +299,7 @@ default_message() ->
                 ?DEFAULT_PRIMARY_STORE,
                 #{
                     <<"store-module">> => hb_store_fs,
-                    <<"name">> => <<"cache-mainnet">>
+                    <<"name">> => <<"cache-mainnet/fs">>
                 },
                 #{
                     <<"store-module">> => hb_store_gateway,

src/hb_store.erl

@@ -46,7 +46,7 @@
 -export([behavior_info/1]).
 -export([start/1, stop/1, reset/1]).
 -export([filter/2, scope/2, sort/2]).
--export([type/2, read/2, write/3, list/2, match/2]).
+-export([type/2, read/2, write/3, sync/2, list/2, match/2]).
 -export([path/1, path/2, add_path/2, add_path/3, join/1]).
 -export([make_group/2, make_link/3, resolve/2]).
 -export([find/1]).
@@ -305,6 +305,71 @@ list(Modules, Path) -> call_function(Modules, list, [Path]).
 %% is given as a list of IDs.
 match(Modules, Match) -> call_function(Modules, match, [Match]).
 
+%% @doc Copies the contents of one store to another.
+sync(#{<<"store-module">> := hb_store_lmdb} = FromStore, ToStore) ->
+    ?event({sync_start, FromStore, ToStore}),
+    FromStoreOpts = maps:put(<<"resolve">>, false, FromStore),
+    Res = hb_store_lmdb:fold_while(FromStoreOpts, fun({Key, Value}, {ok, Acc}) ->
+        case hb_store:write(ToStore, Key, Value) of
+            ok -> 
+                {cont, {ok, Acc + 1}};
+            Error -> 
+                ?event({sync_error, Error}),
+                {halt, {error, sync_failed}} 
+        end
+    end, {ok, 0}),
+    maybe 
+        {ok, Count} ?= Res,
+        ?event({sync_success, Count}),
+        ok
+    end;
+
+sync(#{<<"store-module">> := hb_store_fs} = FromStore, ToStore) ->
+    ?event({sync_start, FromStore, ToStore}),
+    FromStoreOpts = maps:put(<<"resolve">>, false, FromStore),
+    maybe
+        {ok, Entries} ?= hb_store:list(FromStore, <<"/">>),
+        case sync_fs_entries(Entries, <<"/">>, FromStoreOpts, ToStore) of
+            [] -> ok;
+            FailedKeyValues -> {error, {sync_failed, FailedKeyValues}}
+        end
+    end.
+
+sync_fs_entries(Entries, ParentDir, FromStore, ToStore) ->
+    ?event({sync_entries, ParentDir, Entries}),
+    lists:foldl(fun(Key, Acc) ->
+        NewPath =
+            case ParentDir of
+                Bin when Bin == <<"">> orelse Bin == <<"/">> -> Key;
+                _ -> <<ParentDir/binary, "/", Key/binary>>
+            end,
+        case type(FromStore, NewPath) of
+            Type when Type == simple orelse Type == link ->
+                case hb_store:read(FromStore, NewPath) of
+                    {ok, Value} when Type == simple ->
+                        case hb_store:write(ToStore, NewPath, Value) of
+                            ok -> Acc;
+                            _Error -> [{NewPath, Value} | Acc]
+                        end;
+                    {ok, LinkTarget} when Type == link ->
+                        ok = hb_store:make_link(ToStore, LinkTarget, NewPath),
+                        Acc;
+                    _Error ->
+                        [{NewPath, undefined} | Acc]
+                end;
+            composite ->
+                case hb_store:make_group(ToStore, NewPath) of
+                    ok ->
+                        {ok, Entries2} = hb_store:list(FromStore, NewPath),
+                        Acc ++ sync_fs_entries(Entries2, NewPath, FromStore, ToStore);
+                    _Error ->
+                        [{NewPath, undefined} | Acc]
+                end;
+            not_found ->
+                Acc
+        end
+    end, [], Entries).
+
 %% @doc Call a function on the first store module that succeeds. Returns its
 %% result, or `not_found` if none of the stores succeed. If `TIME_CALLS` is set,
 %% this function will also time the call and increment the appropriate event
@@ -411,7 +476,7 @@ call_all(X, _Function, _Args) when not is_list(X) ->
 call_all([], _Function, _Args) ->
     ok;
 call_all([Store = #{<<"store-module">> := Mod} | Rest], Function, Args) ->
-    try apply_store_function(Mod, Function, Store, Args)
+    try apply_store_function(Mod, Store, Function, Args)
     catch
         Class:Reason:Stacktrace ->
             ?event(warning, {store_call_failed, {Class, Reason, Stacktrace}}),
@@ -513,11 +578,144 @@ hierarchical_path_resolution_test(Store) ->
         hb_store:read(Store, [<<"test-link">>, <<"test-file">>])
     ).
 
+%% @doc Test the hb_store:sync function by syncing from hb_store_fs to hb_store_lmdb
+hb_store_sync_test(_Store) ->
+    % Generate unique names to avoid conflicts
+    TestId = integer_to_binary(erlang:system_time(microsecond)),
+    % Set up FromStore (hb_store_fs) with resolve=false as specified
+    FromStore = #{
+        <<"store-module">> => hb_store_fs, 
+        <<"name">> => <<"cache-sync-from-", TestId/binary>>, 
+        <<"resolve">> => false
+    },
+    % Set up ToStore (hb_store_lmdb)
+    ToStore = #{
+        <<"store-module">> => hb_store_lmdb, 
+        <<"name">> => <<"cache-sync-to-", TestId/binary>>
+    },
+
+    % Clean up any existing data
+    hb_store:reset(FromStore),
+    hb_store:reset(ToStore),
+
+    % Start both stores
+    hb_store:start(FromStore),
+    hb_store:start(ToStore),
+
+    % Populate FromStore with directories, files, and links
+    % Create a directory structure
+    ok = hb_store:make_group(FromStore, <<"test-dir">>),
+    ok = hb_store:write(FromStore, <<"test-dir/file1.txt">>, <<"Hello World">>),
+    ok = hb_store:write(FromStore, <<"test-dir/file2.txt">>, <<"Test Data">>),
+
+    % Create a nested directory
+    ok = hb_store:make_group(FromStore, <<"test-dir/nested">>),
+    ok = hb_store:write(FromStore, <<"test-dir/nested/deep-file.txt">>, <<"Deep Content">>),
+
+    % Create some top-level files
+    ok = hb_store:write(FromStore, <<"root-file.txt">>, <<"Root Content">>),
+
+    % Create a link
+    ok = hb_store:make_link(FromStore, <<"root-file.txt">>, <<"link-to-root">>),
+
+    % Perform the sync operation
+    Result = hb_store:sync(FromStore, ToStore),
+    ?assertEqual(ok, Result),
+
+    % Verify that directories exist in ToStore
+    ?assertEqual(composite, hb_store:type(ToStore, <<"test-dir">>)),
+    ?assertEqual(composite, hb_store:type(ToStore, <<"test-dir/nested">>)),
+
+    % Verify that files exist in ToStore
+    {ok, File1Content} = hb_store:read(ToStore, <<"test-dir/file1.txt">>),
+    ?assertEqual(<<"Hello World">>, File1Content),
+
+    {ok, File2Content} = hb_store:read(ToStore, <<"test-dir/file2.txt">>),
+    ?assertEqual(<<"Test Data">>, File2Content),
+
+    {ok, DeepContent} = hb_store:read(ToStore, <<"test-dir/nested/deep-file.txt">>),
+    ?assertEqual(<<"Deep Content">>, DeepContent),
+
+    {ok, RootContent} = hb_store:read(ToStore, <<"root-file.txt">>),
+    ?assertEqual(<<"Root Content">>, RootContent),
+
+    % Verify that links work in ToStore
+    {ok, LinkContent} = hb_store:read(ToStore, <<"link-to-root">>),
+    ?assertEqual(<<"Root Content">>, LinkContent),
+
+    % Clean up
+    hb_store:stop(FromStore),
+    hb_store:stop(ToStore).
+
+%% @doc Test the hb_store:sync function by syncing from hb_store_lmdb to hb_store_lmdb
+hb_store_lmdb_sync_test(_Store) ->
+    % Generate unique names to avoid conflicts
+    TestId = integer_to_binary(erlang:system_time(microsecond)),
+    % Set up FromStore (hb_store_lmdb) with resolve=false as specified
+    FromStore = #{
+        <<"store-module">> => hb_store_lmdb, 
+        <<"name">> => <<"cache-lmdb-sync-from-", TestId/binary>>, 
+        <<"resolve">> => false
+    },
+    % Set up ToStore (hb_store_lmdb)
+    ToStore = #{
+        <<"store-module">> => hb_store_lmdb, 
+        <<"name">> => <<"cache-lmdb-sync-to-", TestId/binary>>
+    },
+
+    % Clean up any existing data
+    hb_store:reset(FromStore),
+    hb_store:reset(ToStore),
+
+    % Start both stores
+    hb_store:start(FromStore),
+    hb_store:start(ToStore),
+
+    % Populate FromStore with data
+    ok = hb_store:write(FromStore, <<"key1">>, <<"value1">>),
+    ok = hb_store:write(FromStore, <<"key2">>, <<"value2">>),
+    ok = hb_store:write(FromStore, <<"nested/key3">>, <<"value3">>),
+    ok = hb_store:write(FromStore, <<"deep/nested/key4">>, <<"value4">>),
+
+    % Create some links
+    ok = hb_store:make_link(FromStore, <<"key1">>, <<"link-to-key1">>),
+    ok = hb_store:make_link(FromStore, <<"nested/key3">>, <<"link-to-nested">>),
+
+    % Perform the sync operation
+    Result = hb_store:sync(FromStore, ToStore),
+    ?assertEqual(ok, Result),
+
+    % Verify that all data exists in ToStore
+    {ok, Value1} = hb_store:read(ToStore, <<"key1">>),
+    ?assertEqual(<<"value1">>, Value1),
+
+    {ok, Value2} = hb_store:read(ToStore, <<"key2">>),
+    ?assertEqual(<<"value2">>, Value2),
+
+    {ok, Value3} = hb_store:read(ToStore, <<"nested/key3">>),
+    ?assertEqual(<<"value3">>, Value3),
+
+    {ok, Value4} = hb_store:read(ToStore, <<"deep/nested/key4">>),
+    ?assertEqual(<<"value4">>, Value4),
+
+    % Verify that links work in ToStore
+    {ok, LinkValue1} = hb_store:read(ToStore, <<"link-to-key1">>),
+    ?assertEqual(<<"value1">>, LinkValue1),
+
+    {ok, LinkValue3} = hb_store:read(ToStore, <<"link-to-nested">>),
+    ?assertEqual(<<"value3">>, LinkValue3),
+
+    % Clean up
+    hb_store:stop(FromStore),
+    hb_store:stop(ToStore).
+
 store_suite_test_() ->
     generate_test_suite([
         {"simple path resolution", fun simple_path_resolution_test/1},
         {"resursive path resolution", fun resursive_path_resolution_test/1},
-        {"hierarchical path resolution", fun hierarchical_path_resolution_test/1}
+        {"hierarchical path resolution", fun hierarchical_path_resolution_test/1},
+        {"hb_store sync", fun hb_store_sync_test/1},
+        {"hb_store lmdb sync", fun hb_store_lmdb_sync_test/1}
     ]).
 
 benchmark_suite_test_() ->