umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Make compression algorithm an option

Browse source 
Compression can be switched between LZ4 and zlib (native).

The setting to determine if compression should happen on receipt is now a macro definition in leveled_codec.
This commit is contained in:
Martin Sumner 2017-11-06 15:54:58 +00:00
parent 4c44e86eab
commit 61b7be5039
9 changed files with 474 additions and 264 deletions
Download patch file Download diff file Expand all files Collapse all files

3
include/leveled.hrl
View file

@ -48,6 +48,7 @@
source_inker :: pid() | undefined, source_inker :: pid() | undefined,
reload_strategy = [] :: list(), reload_strategy = [] :: list(),
waste_retention_period :: integer() | undefined, waste_retention_period :: integer() | undefined,
compression_method :: lz4|native,
max_run_length}). max_run_length}).
-record(penciller_options, -record(penciller_options,
@ -58,6 +59,7 @@
bookies_mem :: tuple() | undefined, bookies_mem :: tuple() | undefined,
source_penciller :: pid() | undefined, source_penciller :: pid() | undefined,
snapshot_longrunning = true :: boolean(), snapshot_longrunning = true :: boolean(),
compression_method :: lz4|native,
levelzero_cointoss = false :: boolean()}). levelzero_cointoss = false :: boolean()}).
-record(iclerk_options, -record(iclerk_options,
@ -65,6 +67,7 @@
max_run_length :: integer() | undefined, max_run_length :: integer() | undefined,
cdb_options = #cdb_options{} :: #cdb_options{}, cdb_options = #cdb_options{} :: #cdb_options{},
waste_retention_period :: integer() | undefined, waste_retention_period :: integer() | undefined,
compression_method :: lz4|native,
reload_strategy = [] :: list()}). reload_strategy = [] :: list()}).
-record(recent_aae, {filter :: whitelist|blacklist, -record(recent_aae, {filter :: whitelist|blacklist,

22
src/leveled_bookie.erl
View file

@ -81,6 +81,7 @@
-define(JOURNAL_SIZE_JITTER, 20). -define(JOURNAL_SIZE_JITTER, 20).
-define(LONG_RUNNING, 80000). -define(LONG_RUNNING, 80000).
-define(RECENT_AAE, false). -define(RECENT_AAE, false).
-define(COMPRESSION_METHOD, lz4).
-record(ledger_cache, {mem :: ets:tab(), -record(ledger_cache, {mem :: ets:tab(),
loader = leveled_tree:empty(?CACHE_TYPE) loader = leveled_tree:empty(?CACHE_TYPE)
@ -387,7 +388,8 @@ init([Opts]) ->
unit_minutes = UnitMinutes} unit_minutes = UnitMinutes}
end, end,
{Inker, Penciller} = startup(InkerOpts, PencillerOpts, RecentAAE), {Inker, Penciller} =
startup(InkerOpts, PencillerOpts, RecentAAE),
NewETS = ets:new(mem, [ordered_set]), NewETS = ets:new(mem, [ordered_set]),
leveled_log:log("B0001", [Inker, Penciller]), leveled_log:log("B0001", [Inker, Penciller]),
@ -911,16 +913,30 @@ set_options(Opts) ->
ok = filelib:ensure_dir(JournalFP), ok = filelib:ensure_dir(JournalFP),
ok = filelib:ensure_dir(LedgerFP), ok = filelib:ensure_dir(LedgerFP),
CompressionMethod =
case get_opt(compression_method, Opts, ?COMPRESSION_METHOD) of
native ->
% Note native compression will have reduced performance
% https://github.com/martinsumner/leveled/issues/95
native;
lz4 ->
% Must include lz4 library in rebar.config
lz4
end,
{#inker_options{root_path = JournalFP, {#inker_options{root_path = JournalFP,
reload_strategy = ReloadStrategy, reload_strategy = ReloadStrategy,
max_run_length = get_opt(max_run_length, Opts), max_run_length = get_opt(max_run_length, Opts),
waste_retention_period = WRP, waste_retention_period = WRP,
cdb_options = #cdb_options{max_size=MaxJournalSize, compression_method = CompressionMethod,
cdb_options =
#cdb_options{max_size=MaxJournalSize,
binary_mode=true, binary_mode=true,
sync_strategy=SyncStrat}}, sync_strategy=SyncStrat}},
#penciller_options{root_path = LedgerFP, #penciller_options{root_path = LedgerFP,
max_inmemory_tablesize = PCLL0CacheSize, max_inmemory_tablesize = PCLL0CacheSize,
levelzero_cointoss = true}}. levelzero_cointoss = true,
compression_method = CompressionMethod}}.
startup(InkerOpts, PencillerOpts, RecentAAE) -> startup(InkerOpts, PencillerOpts, RecentAAE) ->
{ok, Inker} = leveled_inker:ink_start(InkerOpts), {ok, Inker} = leveled_inker:ink_start(InkerOpts),

75
src/leveled_codec.erl
View file

@ -46,7 +46,7 @@
to_ledgerkey/3, to_ledgerkey/3,
to_ledgerkey/5, to_ledgerkey/5,
from_ledgerkey/1, from_ledgerkey/1,
to_inkerkv/4, to_inkerkv/5,
from_inkerkv/1, from_inkerkv/1,
from_inkerkv/2, from_inkerkv/2,
from_journalkey/1, from_journalkey/1,
@ -54,7 +54,7 @@
split_inkvalue/1, split_inkvalue/1,
check_forinkertype/2, check_forinkertype/2,
maybe_compress/1, maybe_compress/1,
create_value_for_journal/2, create_value_for_journal/3,
build_metadata_object/2, build_metadata_object/2,
generate_ledgerkv/5, generate_ledgerkv/5,
get_size/2, get_size/2,
@ -73,6 +73,7 @@
-define(LMD_FORMAT, "~4..0w~2..0w~2..0w~2..0w~2..0w"). -define(LMD_FORMAT, "~4..0w~2..0w~2..0w~2..0w~2..0w").
-define(NRT_IDX, "$aae."). -define(NRT_IDX, "$aae.").
-define(ALL_BUCKETS, <<"$all">>). -define(ALL_BUCKETS, <<"$all">>).
-define(COMPRESS_ON_RECEIPT, true).
-type recent_aae() :: #recent_aae{}. -type recent_aae() :: #recent_aae{}.
-type riak_metadata() :: {binary()|delete, % Sibling Metadata -type riak_metadata() :: {binary()|delete, % Sibling Metadata
@ -214,11 +215,14 @@ to_ledgerkey(Bucket, Key, Tag) ->
%% Return the Key, Value and Hash Option for this object. The hash option %% Return the Key, Value and Hash Option for this object. The hash option
%% indicates whether the key would ever be looked up directly, and so if it %% indicates whether the key would ever be looked up directly, and so if it
%% requires an entry in the hash table %% requires an entry in the hash table
to_inkerkv(LedgerKey, SQN, to_fetch, null) -> to_inkerkv(LedgerKey, SQN, to_fetch, null, _CompressionMethod) ->
{{SQN, ?INKT_STND, LedgerKey}, null, true}; {{SQN, ?INKT_STND, LedgerKey}, null, true};
to_inkerkv(LedgerKey, SQN, Object, KeyChanges) -> to_inkerkv(LedgerKey, SQN, Object, KeyChanges, CompressionMethod) ->
InkerType = check_forinkertype(LedgerKey, Object), InkerType = check_forinkertype(LedgerKey, Object),
Value = create_value_for_journal({Object, KeyChanges}, true), Value =
create_value_for_journal({Object, KeyChanges},
?COMPRESS_ON_RECEIPT,
CompressionMethod),
{{SQN, InkerType, LedgerKey}, Value}. {{SQN, InkerType, LedgerKey}, Value}.
%% Used when fetching objects, so only handles standard, hashable entries %% Used when fetching objects, so only handles standard, hashable entries
@ -235,46 +239,55 @@ from_inkerkv(Object, ToIgnoreKeyChanges) ->
Object Object
end. end.
create_value_for_journal({Object, KeyChanges}, Compress) create_value_for_journal({Object, KeyChanges}, Compress, Method)
when not is_binary(KeyChanges) -> when not is_binary(KeyChanges) ->
KeyChangeBin = term_to_binary(KeyChanges, [compressed]), KeyChangeBin = term_to_binary(KeyChanges, [compressed]),
create_value_for_journal({Object, KeyChangeBin}, Compress); create_value_for_journal({Object, KeyChangeBin}, Compress, Method);
create_value_for_journal({Object, KeyChangeBin}, Compress) -> create_value_for_journal({Object, KeyChangeBin}, Compress, Method) ->
KeyChangeBinLen = byte_size(KeyChangeBin), KeyChangeBinLen = byte_size(KeyChangeBin),
ObjectBin = serialise_object(Object, Compress), ObjectBin = serialise_object(Object, Compress, Method),
TypeCode = encode_valuetype(is_binary(Object), Compress), TypeCode = encode_valuetype(is_binary(Object), Compress, Method),
<<ObjectBin/binary, <<ObjectBin/binary,
KeyChangeBin/binary, KeyChangeBin/binary,
KeyChangeBinLen:32/integer, KeyChangeBinLen:32/integer,
TypeCode:8/integer>>. TypeCode:8/integer>>.
maybe_compress({null, KeyChanges}) -> maybe_compress({null, KeyChanges}) ->
create_value_for_journal({null, KeyChanges}, false); create_value_for_journal({null, KeyChanges}, false, native);
maybe_compress(JournalBin) -> maybe_compress(JournalBin) ->
Length0 = byte_size(JournalBin) - 5, Length0 = byte_size(JournalBin) - 5,
<<JBin0:Length0/binary, <<JBin0:Length0/binary,
KeyChangeLength:32/integer, KeyChangeLength:32/integer,
Type:8/integer>> = JournalBin, Type:8/integer>> = JournalBin,
{IsBinary, IsCompressed} = decode_valuetype(Type), {IsBinary, IsCompressed, IsLz4} = decode_valuetype(Type),
case IsCompressed of case IsCompressed of
true -> true ->
JournalBin; JournalBin;
false -> false ->
Length1 = Length0 - KeyChangeLength, Length1 = Length0 - KeyChangeLength,
<<OBin2:Length1/binary, KCBin2:KeyChangeLength/binary>> = JBin0, <<OBin2:Length1/binary, KCBin2:KeyChangeLength/binary>> = JBin0,
V0 = {deserialise_object(OBin2, IsBinary, IsCompressed), V0 = {deserialise_object(OBin2, IsBinary, IsCompressed, IsLz4),
binary_to_term(KCBin2)}, binary_to_term(KCBin2)},
create_value_for_journal(V0, true) PressMethod = case IsLz4 of
true -> lz4;
false -> native
end,
create_value_for_journal(V0, true, PressMethod)
end. end.
serialise_object(Object, false) when is_binary(Object) -> serialise_object(Object, false, _Method) when is_binary(Object) ->
Object; Object;
serialise_object(Object, true) when is_binary(Object) -> serialise_object(Object, true, Method) when is_binary(Object) ->
case Method of
lz4 ->
{ok, Bin} = lz4:pack(Object), {ok, Bin} = lz4:pack(Object),
Bin; Bin;
serialise_object(Object, false) -> native ->
zlib:compress(Object)
end;
serialise_object(Object, false, _Method) ->
term_to_binary(Object); term_to_binary(Object);
serialise_object(Object, true) -> serialise_object(Object, true, _Method) ->
term_to_binary(Object, [compressed]). term_to_binary(Object, [compressed]).
revert_value_from_journal(JournalBin) -> revert_value_from_journal(JournalBin) ->
@ -285,27 +298,34 @@ revert_value_from_journal(JournalBin, ToIgnoreKeyChanges) ->
<<JBin0:Length0/binary, <<JBin0:Length0/binary,
KeyChangeLength:32/integer, KeyChangeLength:32/integer,
Type:8/integer>> = JournalBin, Type:8/integer>> = JournalBin,
{IsBinary, IsCompressed} = decode_valuetype(Type), {IsBinary, IsCompressed, IsLz4} = decode_valuetype(Type),
Length1 = Length0 - KeyChangeLength, Length1 = Length0 - KeyChangeLength,
case ToIgnoreKeyChanges of case ToIgnoreKeyChanges of
true -> true ->
<<OBin2:Length1/binary, _KCBin2:KeyChangeLength/binary>> = JBin0, <<OBin2:Length1/binary, _KCBin2:KeyChangeLength/binary>> = JBin0,
{deserialise_object(OBin2, IsBinary, IsCompressed), []}; {deserialise_object(OBin2, IsBinary, IsCompressed, IsLz4), []};
false -> false ->
<<OBin2:Length1/binary, KCBin2:KeyChangeLength/binary>> = JBin0, <<OBin2:Length1/binary, KCBin2:KeyChangeLength/binary>> = JBin0,
{deserialise_object(OBin2, IsBinary, IsCompressed), {deserialise_object(OBin2, IsBinary, IsCompressed, IsLz4),
binary_to_term(KCBin2)} binary_to_term(KCBin2)}
end. end.
deserialise_object(Binary, true, true) -> deserialise_object(Binary, true, true, true) ->
{ok, Deflated} = lz4:unpack(Binary), {ok, Deflated} = lz4:unpack(Binary),
Deflated; Deflated;
deserialise_object(Binary, true, false) -> deserialise_object(Binary, true, true, false) ->
zlib:uncompress(Binary);
deserialise_object(Binary, true, false, _IsLz4) ->
Binary; Binary;
deserialise_object(Binary, false, _) -> deserialise_object(Binary, false, _, _IsLz4) ->
binary_to_term(Binary). binary_to_term(Binary).
encode_valuetype(IsBinary, IsCompressed) -> encode_valuetype(IsBinary, IsCompressed, Method) ->
Bit3 =
case Method of
lz4 -> 4;
native -> 0
end,
Bit2 = Bit2 =
case IsBinary of case IsBinary of
true -> 2; true -> 2;
@ -316,12 +336,13 @@ encode_valuetype(IsBinary, IsCompressed) ->
true -> 1; true -> 1;
false -> 0 false -> 0
end, end,
Bit1 + Bit2. Bit1 + Bit2 + Bit3.
decode_valuetype(TypeInt) -> decode_valuetype(TypeInt) ->
IsCompressed = TypeInt band 1 == 1, IsCompressed = TypeInt band 1 == 1,
IsBinary = TypeInt band 2 == 2, IsBinary = TypeInt band 2 == 2,
{IsBinary, IsCompressed}. IsLz4 = TypeInt band 4 ==4,
{IsBinary, IsCompressed, IsLz4}.
from_journalkey({SQN, _Type, LedgerKey}) -> from_journalkey({SQN, _Type, LedgerKey}) ->
{SQN, LedgerKey}. {SQN, LedgerKey}.

15
src/leveled_iclerk.erl
View file

@ -109,7 +109,8 @@
cdb_options, cdb_options,
waste_retention_period :: integer() | undefined, waste_retention_period :: integer() | undefined,
waste_path :: string() | undefined, waste_path :: string() | undefined,
reload_strategy = ?DEFAULT_RELOAD_STRATEGY :: list()}). reload_strategy = ?DEFAULT_RELOAD_STRATEGY :: list(),
compression_method :: lz4|native}).
-record(candidate, {low_sqn :: integer() | undefined, -record(candidate, {low_sqn :: integer() | undefined,
filename :: string() | undefined, filename :: string() | undefined,
@ -167,7 +168,9 @@ init([IClerkOpts]) ->
cdb_options = CDBopts, cdb_options = CDBopts,
reload_strategy = ReloadStrategy, reload_strategy = ReloadStrategy,
waste_path = WP, waste_path = WP,
waste_retention_period = WRP}}. waste_retention_period = WRP,
compression_method =
IClerkOpts#iclerk_options.compression_method}}.
handle_call(_Msg, _From, State) -> handle_call(_Msg, _From, State) ->
{reply, not_supported, State}. {reply, not_supported, State}.
@ -754,7 +757,7 @@ test_ledgerkey(Key) ->
{o, "Bucket", Key, null}. {o, "Bucket", Key, null}.
test_inkerkv(SQN, Key, V, IdxSpecs) -> test_inkerkv(SQN, Key, V, IdxSpecs) ->
leveled_codec:to_inkerkv(test_ledgerkey(Key), SQN, V, IdxSpecs). leveled_codec:to_inkerkv(test_ledgerkey(Key), SQN, V, IdxSpecs, native).
fetch_testcdb(RP) -> fetch_testcdb(RP) ->
FN1 = leveled_inker:filepath(RP, 1, new_journal), FN1 = leveled_inker:filepath(RP, 1, new_journal),
@ -936,13 +939,15 @@ compact_singlefile_totwosmallfiles_testto() ->
lists:foreach(fun(X) -> lists:foreach(fun(X) ->
LK = test_ledgerkey("Key" ++ integer_to_list(X)), LK = test_ledgerkey("Key" ++ integer_to_list(X)),
Value = leveled_rand:rand_bytes(1024), Value = leveled_rand:rand_bytes(1024),
{IK, IV} = leveled_codec:to_inkerkv(LK, X, Value, []), {IK, IV} =
leveled_codec:to_inkerkv(LK, X, Value, [], native),
ok = leveled_cdb:cdb_put(CDB1, IK, IV) ok = leveled_cdb:cdb_put(CDB1, IK, IV)
end, end,
lists:seq(1, 1000)), lists:seq(1, 1000)),
{ok, NewName} = leveled_cdb:cdb_complete(CDB1), {ok, NewName} = leveled_cdb:cdb_complete(CDB1),
{ok, CDBr} = leveled_cdb:cdb_open_reader(NewName), {ok, CDBr} = leveled_cdb:cdb_open_reader(NewName),
CDBoptsSmall = #cdb_options{binary_mode=true, max_size=400000, file_path=CP}, CDBoptsSmall =
#cdb_options{binary_mode=true, max_size=400000, file_path=CP},
BestRun1 = [#candidate{low_sqn=1, BestRun1 = [#candidate{low_sqn=1,
filename=leveled_cdb:cdb_filename(CDBr), filename=leveled_cdb:cdb_filename(CDBr),
journal=CDBr, journal=CDBr,

39
src/leveled_inker.erl
View file

@ -136,6 +136,7 @@
clerk :: pid() | undefined, clerk :: pid() | undefined,
compaction_pending = false :: boolean(), compaction_pending = false :: boolean(),
is_snapshot = false :: boolean(), is_snapshot = false :: boolean(),
compression_method :: lz4|native,
source_inker :: pid() | undefined}). source_inker :: pid() | undefined}).
@ -509,10 +510,12 @@ start_from_file(InkOpts) ->
ReloadStrategy = InkOpts#inker_options.reload_strategy, ReloadStrategy = InkOpts#inker_options.reload_strategy,
MRL = InkOpts#inker_options.max_run_length, MRL = InkOpts#inker_options.max_run_length,
WRP = InkOpts#inker_options.waste_retention_period, WRP = InkOpts#inker_options.waste_retention_period,
Compression = InkOpts#inker_options.compression_method,
IClerkOpts = #iclerk_options{inker = self(), IClerkOpts = #iclerk_options{inker = self(),
cdb_options=IClerkCDBOpts, cdb_options=IClerkCDBOpts,
waste_retention_period = WRP, waste_retention_period = WRP,
reload_strategy = ReloadStrategy, reload_strategy = ReloadStrategy,
compression_method = Compression,
max_run_length = MRL}, max_run_length = MRL},
{ok, Clerk} = leveled_iclerk:clerk_new(IClerkOpts), {ok, Clerk} = leveled_iclerk:clerk_new(IClerkOpts),
@ -528,16 +531,19 @@ start_from_file(InkOpts) ->
active_journaldb = ActiveJournal, active_journaldb = ActiveJournal,
root_path = RootPath, root_path = RootPath,
cdb_options = CDBopts#cdb_options{waste_path=WasteFP}, cdb_options = CDBopts#cdb_options{waste_path=WasteFP},
compression_method = Compression,
clerk = Clerk}}. clerk = Clerk}}.
put_object(LedgerKey, Object, KeyChanges, State) -> put_object(LedgerKey, Object, KeyChanges, State) ->
NewSQN = State#state.journal_sqn + 1, NewSQN = State#state.journal_sqn + 1,
ActiveJournal = State#state.active_journaldb, ActiveJournal = State#state.active_journaldb,
{JournalKey, JournalBin} = leveled_codec:to_inkerkv(LedgerKey, {JournalKey, JournalBin} =
leveled_codec:to_inkerkv(LedgerKey,
NewSQN, NewSQN,
Object, Object,
KeyChanges), KeyChanges,
State#state.compression_method),
case leveled_cdb:cdb_put(ActiveJournal, case leveled_cdb:cdb_put(ActiveJournal,
JournalKey, JournalKey,
JournalBin) of JournalBin) of
@ -579,19 +585,23 @@ get_object(LedgerKey, SQN, Manifest) ->
get_object(LedgerKey, SQN, Manifest, ToIgnoreKeyChanges) -> get_object(LedgerKey, SQN, Manifest, ToIgnoreKeyChanges) ->
JournalP = leveled_imanifest:find_entry(SQN, Manifest), JournalP = leveled_imanifest:find_entry(SQN, Manifest),
{InkerKey, _V, true} = leveled_codec:to_inkerkv(LedgerKey, {InkerKey, _V, true} =
leveled_codec:to_inkerkv(LedgerKey,
SQN, SQN,
to_fetch, to_fetch,
null), null,
not_applicable),
Obj = leveled_cdb:cdb_get(JournalP, InkerKey), Obj = leveled_cdb:cdb_get(JournalP, InkerKey),
leveled_codec:from_inkerkv(Obj, ToIgnoreKeyChanges). leveled_codec:from_inkerkv(Obj, ToIgnoreKeyChanges).
key_check(LedgerKey, SQN, Manifest) -> key_check(LedgerKey, SQN, Manifest) ->
JournalP = leveled_imanifest:find_entry(SQN, Manifest), JournalP = leveled_imanifest:find_entry(SQN, Manifest),
{InkerKey, _V, true} = leveled_codec:to_inkerkv(LedgerKey, {InkerKey, _V, true} =
leveled_codec:to_inkerkv(LedgerKey,
SQN, SQN,
to_fetch, to_fetch,
null), null,
not_applicable),
leveled_cdb:cdb_keycheck(JournalP, InkerKey). leveled_cdb:cdb_keycheck(JournalP, InkerKey).
build_manifest(ManifestFilenames, build_manifest(ManifestFilenames,
@ -851,7 +861,7 @@ initiate_penciller_snapshot(Bookie) ->
-ifdef(TEST). -ifdef(TEST).
create_value_for_journal(Obj, Comp) -> create_value_for_journal(Obj, Comp) ->
leveled_codec:create_value_for_journal(Obj, Comp). leveled_codec:create_value_for_journal(Obj, Comp, native).
build_dummy_journal() -> build_dummy_journal() ->
F = fun(X) -> X end, F = fun(X) -> X end,
@ -933,7 +943,8 @@ simple_inker_test() ->
build_dummy_journal(), build_dummy_journal(),
CDBopts = #cdb_options{max_size=300000, binary_mode=true}, CDBopts = #cdb_options{max_size=300000, binary_mode=true},
{ok, Ink1} = ink_start(#inker_options{root_path=RootPath, {ok, Ink1} = ink_start(#inker_options{root_path=RootPath,
cdb_options=CDBopts}), cdb_options=CDBopts,
compression_method=native}),
Obj1 = ink_get(Ink1, "Key1", 1), Obj1 = ink_get(Ink1, "Key1", 1),
?assertMatch({{1, "Key1"}, {"TestValue1", []}}, Obj1), ?assertMatch({{1, "Key1"}, {"TestValue1", []}}, Obj1),
Obj3 = ink_get(Ink1, "Key1", 3), Obj3 = ink_get(Ink1, "Key1", 3),
@ -955,7 +966,8 @@ simple_inker_completeactivejournal_test() ->
F1r = filename:join(JournalFP, "nursery_1.pnd"), F1r = filename:join(JournalFP, "nursery_1.pnd"),
ok = file:rename(F1, F1r), ok = file:rename(F1, F1r),
{ok, Ink1} = ink_start(#inker_options{root_path=RootPath, {ok, Ink1} = ink_start(#inker_options{root_path=RootPath,
cdb_options=CDBopts}), cdb_options=CDBopts,
compression_method=native}),
Obj1 = ink_get(Ink1, "Key1", 1), Obj1 = ink_get(Ink1, "Key1", 1),
?assertMatch({{1, "Key1"}, {"TestValue1", []}}, Obj1), ?assertMatch({{1, "Key1"}, {"TestValue1", []}}, Obj1),
Obj2 = ink_get(Ink1, "Key4", 4), Obj2 = ink_get(Ink1, "Key4", 4),
@ -973,7 +985,8 @@ compact_journal_test() ->
RStrategy = [{?STD_TAG, recovr}], RStrategy = [{?STD_TAG, recovr}],
{ok, Ink1} = ink_start(#inker_options{root_path=RootPath, {ok, Ink1} = ink_start(#inker_options{root_path=RootPath,
cdb_options=CDBopts, cdb_options=CDBopts,
reload_strategy=RStrategy}), reload_strategy=RStrategy,
compression_method=native}),
{ok, NewSQN1, _ObjSize} = ink_put(Ink1, {ok, NewSQN1, _ObjSize} = ink_put(Ink1,
test_ledgerkey("KeyAA"), test_ledgerkey("KeyAA"),
"TestValueAA", "TestValueAA",
@ -1039,7 +1052,8 @@ empty_manifest_test() ->
clean_testdir(RootPath), clean_testdir(RootPath),
CDBopts = #cdb_options{max_size=300000}, CDBopts = #cdb_options{max_size=300000},
{ok, Ink1} = ink_start(#inker_options{root_path=RootPath, {ok, Ink1} = ink_start(#inker_options{root_path=RootPath,
cdb_options=CDBopts}), cdb_options=CDBopts,
compression_method=native}),
?assertMatch(not_present, ink_fetch(Ink1, "Key1", 1)), ?assertMatch(not_present, ink_fetch(Ink1, "Key1", 1)),
CheckFun = fun(L, K, SQN) -> lists:member({SQN, K}, L) end, CheckFun = fun(L, K, SQN) -> lists:member({SQN, K}, L) end,
@ -1059,7 +1073,8 @@ empty_manifest_test() ->
ok = file:write_file(FN, term_to_binary("Hello")), ok = file:write_file(FN, term_to_binary("Hello")),
{ok, Ink2} = ink_start(#inker_options{root_path=RootPath, {ok, Ink2} = ink_start(#inker_options{root_path=RootPath,
cdb_options=CDBopts}), cdb_options=CDBopts,
compression_method=native}),
?assertMatch(not_present, ink_fetch(Ink2, "Key1", 1)), ?assertMatch(not_present, ink_fetch(Ink2, "Key1", 1)),
{ok, SQN, Size} = ink_put(Ink2, "Key1", "Value1", {[], infinity}), {ok, SQN, Size} = ink_put(Ink2, "Key1", "Value1", {[], infinity}),
?assertMatch(2, SQN), ?assertMatch(2, SQN),

60
src/leveled_pclerk.erl
View file

@ -35,7 +35,7 @@
]). ]).
-export([ -export([
clerk_new/2, clerk_new/3,
clerk_prompt/1, clerk_prompt/1,
clerk_push/2, clerk_push/2,
clerk_close/1, clerk_close/1,
@ -49,15 +49,19 @@
-record(state, {owner :: pid() | undefined, -record(state, {owner :: pid() | undefined,
root_path :: string() | undefined, root_path :: string() | undefined,
pending_deletions = dict:new() % OTP 16 does not like type pending_deletions = dict:new(), % OTP 16 does not like type
compression_method :: lz4|native
}). }).
%%%============================================================================ %%%============================================================================
%%% API %%% API
%%%============================================================================ %%%============================================================================
clerk_new(Owner, Manifest) -> clerk_new(Owner, Manifest, CompressionMethod) ->
{ok, Pid} = gen_server:start(?MODULE, [], []), {ok, Pid} =
gen_server:start(?MODULE,
[{compression_method, CompressionMethod}],
[]),
ok = gen_server:call(Pid, {load, Owner, Manifest}, infinity), ok = gen_server:call(Pid, {load, Owner, Manifest}, infinity),
leveled_log:log("PC001", [Pid, Owner]), leveled_log:log("PC001", [Pid, Owner]),
{ok, Pid}. {ok, Pid}.
@ -78,8 +82,8 @@ clerk_close(Pid) ->
%%% gen_server callbacks %%% gen_server callbacks
%%%============================================================================ %%%============================================================================
init([]) -> init([{compression_method, CompressionMethod}]) ->
{ok, #state{}}. {ok, #state{compression_method = CompressionMethod}}.
handle_call({load, Owner, RootPath}, _From, State) -> handle_call({load, Owner, RootPath}, _From, State) ->
{reply, ok, State#state{owner=Owner, root_path=RootPath}, ?MIN_TIMEOUT}; {reply, ok, State#state{owner=Owner, root_path=RootPath}, ?MIN_TIMEOUT};
@ -120,7 +124,8 @@ request_work(State) ->
handle_work({SrcLevel, Manifest}, State) -> handle_work({SrcLevel, Manifest}, State) ->
{UpdManifest, EntriesToDelete} = merge(SrcLevel, {UpdManifest, EntriesToDelete} = merge(SrcLevel,
Manifest, Manifest,
State#state.root_path), State#state.root_path,
State#state.compression_method),
leveled_log:log("PC007", []), leveled_log:log("PC007", []),
SWMC = os:timestamp(), SWMC = os:timestamp(),
ok = leveled_penciller:pcl_manifestchange(State#state.owner, ok = leveled_penciller:pcl_manifestchange(State#state.owner,
@ -132,7 +137,7 @@ handle_work({SrcLevel, Manifest}, State) ->
leveled_log:log_timer("PC018", [], SWSM), leveled_log:log_timer("PC018", [], SWSM),
{leveled_pmanifest:get_manifest_sqn(UpdManifest), EntriesToDelete}. {leveled_pmanifest:get_manifest_sqn(UpdManifest), EntriesToDelete}.
merge(SrcLevel, Manifest, RootPath) -> merge(SrcLevel, Manifest, RootPath, CompressionMethod) ->
Src = leveled_pmanifest:mergefile_selector(Manifest, SrcLevel), Src = leveled_pmanifest:mergefile_selector(Manifest, SrcLevel),
NewSQN = leveled_pmanifest:get_manifest_sqn(Manifest) + 1, NewSQN = leveled_pmanifest:get_manifest_sqn(Manifest) + 1,
SinkList = leveled_pmanifest:merge_lookup(Manifest, SinkList = leveled_pmanifest:merge_lookup(Manifest,
@ -152,7 +157,9 @@ merge(SrcLevel, Manifest, RootPath) ->
{Man0, []}; {Man0, []};
_ -> _ ->
SST_RP = leveled_penciller:sst_rootpath(RootPath), SST_RP = leveled_penciller:sst_rootpath(RootPath),
perform_merge(Manifest, Src, SinkList, SrcLevel, SST_RP, NewSQN) perform_merge(Manifest,
Src, SinkList, SrcLevel,
SST_RP, NewSQN, CompressionMethod)
end. end.
notify_deletions([], _Penciller) -> notify_deletions([], _Penciller) ->
@ -167,15 +174,20 @@ notify_deletions([Head|Tail], Penciller) ->
%% %%
%% SrcLevel is the level of the src sst file, the sink should be srcLevel + 1 %% SrcLevel is the level of the src sst file, the sink should be srcLevel + 1
perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) -> perform_merge(Manifest,
Src, SinkList, SrcLevel,
RootPath, NewSQN,
CompressionMethod) ->
leveled_log:log("PC010", [Src#manifest_entry.filename, NewSQN]), leveled_log:log("PC010", [Src#manifest_entry.filename, NewSQN]),
SrcList = [{next, Src, all}], SrcList = [{next, Src, all}],
MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner),
SinkLevel = SrcLevel + 1, SinkLevel = SrcLevel + 1,
SinkBasement = leveled_pmanifest:is_basement(Manifest, SinkLevel), SinkBasement = leveled_pmanifest:is_basement(Manifest, SinkLevel),
Additions = do_merge(SrcList, SinkList, Additions =
do_merge(SrcList, SinkList,
SinkLevel, SinkBasement, SinkLevel, SinkBasement,
RootPath, NewSQN, MaxSQN, RootPath, NewSQN, MaxSQN,
CompressionMethod,
[]), []),
RevertPointerFun = RevertPointerFun =
fun({next, ME, _SK}) -> fun({next, ME, _SK}) ->
@ -193,22 +205,23 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) ->
Src), Src),
{Man2, [Src|SinkManifestList]}. {Man2, [Src|SinkManifestList]}.
do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, Additions) -> do_merge([], [], SinkLevel, _SinkB, _RP, NewSQN, _MaxSQN, _CM, Additions) ->
leveled_log:log("PC011", [NewSQN, SinkLevel, length(Additions)]), leveled_log:log("PC011", [NewSQN, SinkLevel, length(Additions)]),
Additions; Additions;
do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, Additions) -> do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, CM, Additions) ->
FileName = leveled_penciller:sst_filename(NewSQN, FileName = leveled_penciller:sst_filename(NewSQN,
SinkLevel, SinkLevel,
length(Additions)), length(Additions)),
leveled_log:log("PC012", [NewSQN, FileName, SinkB]), leveled_log:log("PC012", [NewSQN, FileName, SinkB]),
TS1 = os:timestamp(), TS1 = os:timestamp(),
case leveled_sst:sst_new(RP, FileName, case leveled_sst:sst_new(RP, FileName,
KL1, KL2, SinkB, SinkLevel, MaxSQN) of KL1, KL2, SinkB, SinkLevel, MaxSQN, CM) of
empty -> empty ->
leveled_log:log("PC013", [FileName]), leveled_log:log("PC013", [FileName]),
do_merge([], [], do_merge([], [],
SinkLevel, SinkB, SinkLevel, SinkB,
RP, NewSQN, MaxSQN, RP, NewSQN, MaxSQN,
CM,
Additions); Additions);
{ok, Pid, Reply} -> {ok, Pid, Reply} ->
{{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply, {{KL1Rem, KL2Rem}, SmallestKey, HighestKey} = Reply,
@ -220,6 +233,7 @@ do_merge(KL1, KL2, SinkLevel, SinkB, RP, NewSQN, MaxSQN, Additions) ->
do_merge(KL1Rem, KL2Rem, do_merge(KL1Rem, KL2Rem,
SinkLevel, SinkB, SinkLevel, SinkB,
RP, NewSQN, MaxSQN, RP, NewSQN, MaxSQN,
CM,
Additions ++ [Entry]) Additions ++ [Entry])
end. end.
@ -265,31 +279,36 @@ merge_file_test() ->
"KL1_L1.sst", "KL1_L1.sst",
1, 1,
KL1_L1, KL1_L1,
999999), 999999,
native),
KL1_L2 = lists:sort(generate_randomkeys(8000, 0, 250)), KL1_L2 = lists:sort(generate_randomkeys(8000, 0, 250)),
{ok, PidL2_1, _} = leveled_sst:sst_new("../test/", {ok, PidL2_1, _} = leveled_sst:sst_new("../test/",
"KL1_L2.sst", "KL1_L2.sst",
2, 2,
KL1_L2, KL1_L2,
999999), 999999,
native),
KL2_L2 = lists:sort(generate_randomkeys(8000, 250, 250)), KL2_L2 = lists:sort(generate_randomkeys(8000, 250, 250)),
{ok, PidL2_2, _} = leveled_sst:sst_new("../test/", {ok, PidL2_2, _} = leveled_sst:sst_new("../test/",
"KL2_L2.sst", "KL2_L2.sst",
2, 2,
KL2_L2, KL2_L2,
999999), 999999,
lz4),
KL3_L2 = lists:sort(generate_randomkeys(8000, 500, 250)), KL3_L2 = lists:sort(generate_randomkeys(8000, 500, 250)),
{ok, PidL2_3, _} = leveled_sst:sst_new("../test/", {ok, PidL2_3, _} = leveled_sst:sst_new("../test/",
"KL3_L2.sst", "KL3_L2.sst",
2, 2,
KL3_L2, KL3_L2,
999999), 999999,
lz4),
KL4_L2 = lists:sort(generate_randomkeys(8000, 750, 250)), KL4_L2 = lists:sort(generate_randomkeys(8000, 750, 250)),
{ok, PidL2_4, _} = leveled_sst:sst_new("../test/", {ok, PidL2_4, _} = leveled_sst:sst_new("../test/",
"KL4_L2.sst", "KL4_L2.sst",
2, 2,
KL4_L2, KL4_L2,
999999), 999999,
lz4),
E1 = #manifest_entry{owner = PidL1_1, E1 = #manifest_entry{owner = PidL1_1,
filename = "./KL1_L1.sst", filename = "./KL1_L1.sst",
@ -321,7 +340,8 @@ merge_file_test() ->
PointerList = lists:map(fun(ME) -> {next, ME, all} end, PointerList = lists:map(fun(ME) -> {next, ME, all} end,
[E2, E3, E4, E5]), [E2, E3, E4, E5]),
{Man6, _Dels} = perform_merge(Man5, E1, PointerList, 1, "../test", 3), {Man6, _Dels} =
perform_merge(Man5, E1, PointerList, 1, "../test", 3, native),
?assertMatch(3, leveled_pmanifest:get_manifest_sqn(Man6)). ?assertMatch(3, leveled_pmanifest:get_manifest_sqn(Man6)).

39
src/leveled_penciller.erl
View file

@ -243,7 +243,9 @@
work_ongoing = false :: boolean(), % i.e. compaction work work_ongoing = false :: boolean(), % i.e. compaction work
work_backlog = false :: boolean(), % i.e. compaction work work_backlog = false :: boolean(), % i.e. compaction work
head_timing :: tuple() | undefined}). head_timing :: tuple() | undefined,
compression_method :: lz4|native}).
-type penciller_options() :: #penciller_options{}. -type penciller_options() :: #penciller_options{}.
-type bookies_memory() :: {tuple()|empty_cache, -type bookies_memory() :: {tuple()|empty_cache,
@ -835,25 +837,28 @@ sst_filename(ManSQN, Level, Count) ->
start_from_file(PCLopts) -> start_from_file(PCLopts) ->
RootPath = PCLopts#penciller_options.root_path, RootPath = PCLopts#penciller_options.root_path,
MaxTableSize = case PCLopts#penciller_options.max_inmemory_tablesize of MaxTableSize =
case PCLopts#penciller_options.max_inmemory_tablesize of
undefined -> undefined ->
?MAX_TABLESIZE; ?MAX_TABLESIZE;
M -> M ->
M M
end, end,
PressMethod = PCLopts#penciller_options.compression_method,
{ok, MergeClerk} = leveled_pclerk:clerk_new(self(), RootPath), {ok, MergeClerk} = leveled_pclerk:clerk_new(self(), RootPath, PressMethod),
CoinToss = PCLopts#penciller_options.levelzero_cointoss, CoinToss = PCLopts#penciller_options.levelzero_cointoss,
% Used to randomly defer the writing of L0 file. Intended to help with % Used to randomly defer the writing of L0 file. Intended to help with
% vnode syncronisation issues (e.g. stop them all by default merging to % vnode syncronisation issues (e.g. stop them all by default merging to
% level zero concurrently) % level zero concurrently)
InitState = #state{clerk=MergeClerk, InitState = #state{clerk = MergeClerk,
root_path=RootPath, root_path = RootPath,
levelzero_maxcachesize=MaxTableSize, levelzero_maxcachesize = MaxTableSize,
levelzero_cointoss=CoinToss, levelzero_cointoss = CoinToss,
levelzero_index=leveled_pmem:new_index()}, levelzero_index = leveled_pmem:new_index(),
compression_method = PressMethod},
%% Open manifest %% Open manifest
Manifest0 = leveled_pmanifest:open_manifest(RootPath), Manifest0 = leveled_pmanifest:open_manifest(RootPath),
@ -861,7 +866,8 @@ start_from_file(PCLopts) ->
fun(FN) -> fun(FN) ->
{ok, {ok,
Pid, Pid,
{_FK, _LK}} = leveled_sst:sst_open(sst_rootpath(RootPath), FN), {_FK, _LK}} =
leveled_sst:sst_open(sst_rootpath(RootPath), FN),
Pid Pid
end, end,
SQNFun = fun leveled_sst:sst_getmaxsequencenumber/1, SQNFun = fun leveled_sst:sst_getmaxsequencenumber/1,
@ -1006,7 +1012,8 @@ roll_memory(State, false) ->
length(State#state.levelzero_cache), length(State#state.levelzero_cache),
FetchFun, FetchFun,
PCL, PCL,
State#state.ledger_sqn), State#state.ledger_sqn,
State#state.compression_method),
{ok, Constructor, _} = R, {ok, Constructor, _} = R,
Constructor; Constructor;
roll_memory(State, true) -> roll_memory(State, true) ->
@ -1020,7 +1027,8 @@ roll_memory(State, true) ->
FileName, FileName,
0, 0,
KVList, KVList,
State#state.ledger_sqn), State#state.ledger_sqn,
State#state.compression_method),
{ok, Constructor, _} = R, {ok, Constructor, _} = R,
Constructor. Constructor.
@ -1401,7 +1409,8 @@ simple_server_test() ->
RootPath = "../test/ledger", RootPath = "../test/ledger",
clean_testdir(RootPath), clean_testdir(RootPath),
{ok, PCL} = pcl_start(#penciller_options{root_path=RootPath, {ok, PCL} = pcl_start(#penciller_options{root_path=RootPath,
max_inmemory_tablesize=1000}), max_inmemory_tablesize=1000,
compression_method=native}),
Key1_Pre = {{o,"Bucket0001", "Key0001", null}, Key1_Pre = {{o,"Bucket0001", "Key0001", null},
{1, {active, infinity}, null}}, {1, {active, infinity}, null}},
Key1 = add_missing_hash(Key1_Pre), Key1 = add_missing_hash(Key1_Pre),
@ -1440,7 +1449,8 @@ simple_server_test() ->
ok = pcl_close(PCL), ok = pcl_close(PCL),
{ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath, {ok, PCLr} = pcl_start(#penciller_options{root_path=RootPath,
max_inmemory_tablesize=1000}), max_inmemory_tablesize=1000,
compression_method=native}),
?assertMatch(2003, pcl_getstartupsequencenumber(PCLr)), ?assertMatch(2003, pcl_getstartupsequencenumber(PCLr)),
% ok = maybe_pause_push(PCLr, [Key2] ++ KL2 ++ [Key3]), % ok = maybe_pause_push(PCLr, [Key2] ++ KL2 ++ [Key3]),
@ -1689,7 +1699,8 @@ create_file_test() ->
1, 1,
FetchFun, FetchFun,
undefined, undefined,
10000), 10000,
native),
lists:foreach(fun(X) -> lists:foreach(fun(X) ->
case checkready(SP) of case checkready(SP) of
timeout -> timeout ->

379
src/leveled_sst.erl
View file

@ -91,9 +91,9 @@
delete_pending/2, delete_pending/2,
delete_pending/3]). delete_pending/3]).
-export([sst_new/5, -export([sst_new/6,
sst_new/7, sst_new/8,
sst_newlevelzero/6, sst_newlevelzero/7,
sst_open/2, sst_open/2,
sst_get/2, sst_get/2,
sst_get/3, sst_get/3,
@ -127,14 +127,16 @@
%% see Issue 52. Handling within the SST process may lead to contention and %% see Issue 52. Handling within the SST process may lead to contention and
%% extra copying. Files at the top of the tree yield, those lower down don't. %% extra copying. Files at the top of the tree yield, those lower down don't.
-record(state, {summary, -record(state,
{summary,
handle :: file:fd() | undefined, handle :: file:fd() | undefined,
sst_timings :: tuple() | undefined, sst_timings :: tuple() | undefined,
penciller :: pid() | undefined, penciller :: pid() | undefined,
root_path, root_path,
filename, filename,
yield_blockquery = false :: boolean(), yield_blockquery = false :: boolean(),
blockindex_cache}). blockindex_cache,
compression_method :: lz4|native}).
%%%============================================================================ %%%============================================================================
@ -159,29 +161,30 @@ sst_open(RootPath, Filename) ->
{ok, Pid, {SK, EK}} {ok, Pid, {SK, EK}}
end. end.
-spec sst_new(string(), string(), integer(), list(), integer()) -> -spec sst_new(string(), string(), integer(), list(), integer(), lz4|native) ->
{ok, pid(), {tuple(), tuple()}}. {ok, pid(), {tuple(), tuple()}}.
%% @doc %% @doc
%% Start a new SST file at the assigned level passing in a list of Key, Value %% Start a new SST file at the assigned level passing in a list of Key, Value
%% pairs. This should not be used for basement levels or unexpanded Key/Value %% pairs. This should not be used for basement levels or unexpanded Key/Value
%% lists as merge_lists will not be called. %% lists as merge_lists will not be called.
sst_new(RootPath, Filename, Level, KVList, MaxSQN) -> sst_new(RootPath, Filename, Level, KVList, MaxSQN, PressMethod) ->
{ok, Pid} = gen_fsm:start(?MODULE, [], []), {ok, Pid} = gen_fsm:start(?MODULE, [], []),
{[], [], SlotList, FK} = merge_lists(KVList), {[], [], SlotList, FK} = merge_lists(KVList, PressMethod),
case gen_fsm:sync_send_event(Pid, case gen_fsm:sync_send_event(Pid,
{sst_new, {sst_new,
RootPath, RootPath,
Filename, Filename,
Level, Level,
{SlotList, FK}, {SlotList, FK},
MaxSQN}, MaxSQN,
PressMethod},
infinity) of infinity) of
{ok, {SK, EK}} -> {ok, {SK, EK}} ->
{ok, Pid, {SK, EK}} {ok, Pid, {SK, EK}}
end. end.
-spec sst_new(string(), string(), list(), list(), -spec sst_new(string(), string(), list(), list(),
boolean(), integer(), integer()) -> boolean(), integer(), integer(), lz4|native) ->
empty|{ok, pid(), {{list(), list()}, tuple(), tuple()}}. empty|{ok, pid(), {{list(), list()}, tuple(), tuple()}}.
%% @doc %% @doc
%% Start a new SST file at the assigned level passing in a two lists of %% Start a new SST file at the assigned level passing in a two lists of
@ -194,8 +197,11 @@ sst_new(RootPath, Filename, Level, KVList, MaxSQN) ->
%% be that the merge_lists returns nothin (for example when a basement file is %% be that the merge_lists returns nothin (for example when a basement file is
%% all tombstones) - and the atome empty is returned in this case so that the %% all tombstones) - and the atome empty is returned in this case so that the
%% file is not added to the manifest. %% file is not added to the manifest.
sst_new(RootPath, Filename, KVL1, KVL2, IsBasement, Level, MaxSQN) -> sst_new(RootPath, Filename,
{Rem1, Rem2, SlotList, FK} = merge_lists(KVL1, KVL2, {IsBasement, Level}), KVL1, KVL2, IsBasement, Level,
MaxSQN, PressMethod) ->
{Rem1, Rem2, SlotList, FK} =
merge_lists(KVL1, KVL2, {IsBasement, Level}, PressMethod),
case SlotList of case SlotList of
[] -> [] ->
empty; empty;
@ -207,7 +213,8 @@ sst_new(RootPath, Filename, KVL1, KVL2, IsBasement, Level, MaxSQN) ->
Filename, Filename,
Level, Level,
{SlotList, FK}, {SlotList, FK},
MaxSQN}, MaxSQN,
PressMethod},
infinity) of infinity) of
{ok, {SK, EK}} -> {ok, {SK, EK}} ->
{ok, Pid, {{Rem1, Rem2}, SK, EK}} {ok, Pid, {{Rem1, Rem2}, SK, EK}}
@ -215,13 +222,16 @@ sst_new(RootPath, Filename, KVL1, KVL2, IsBasement, Level, MaxSQN) ->
end. end.
-spec sst_newlevelzero(string(), string(), -spec sst_newlevelzero(string(), string(),
integer(), fun(), pid()|undefined, integer()) -> integer(), fun(), pid()|undefined, integer(),
lz4|native) ->
{ok, pid(), noreply}. {ok, pid(), noreply}.
%% @doc %% @doc
%% Start a new file at level zero. At this level the file size is not fixed - %% Start a new file at level zero. At this level the file size is not fixed -
%% it will be as big as the input. Also the KVList is not passed in, it is %% it will be as big as the input. Also the KVList is not passed in, it is
%% fetched slot by slot using the FetchFun %% fetched slot by slot using the FetchFun
sst_newlevelzero(RootPath, Filename, Slots, FetchFun, Penciller, MaxSQN) -> sst_newlevelzero(RootPath, Filename,
Slots, FetchFun, Penciller,
MaxSQN, PressMethod) ->
{ok, Pid} = gen_fsm:start(?MODULE, [], []), {ok, Pid} = gen_fsm:start(?MODULE, [], []),
gen_fsm:send_event(Pid, gen_fsm:send_event(Pid,
{sst_newlevelzero, {sst_newlevelzero,
@ -230,7 +240,8 @@ sst_newlevelzero(RootPath, Filename, Slots, FetchFun, Penciller, MaxSQN) ->
Slots, Slots,
FetchFun, FetchFun,
Penciller, Penciller,
MaxSQN}), MaxSQN,
PressMethod}),
{ok, Pid, noreply}. {ok, Pid, noreply}.
-spec sst_get(pid(), tuple()) -> tuple()|not_present. -spec sst_get(pid(), tuple()) -> tuple()|not_present.
@ -261,10 +272,10 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) ->
case gen_fsm:sync_send_event(Pid, case gen_fsm:sync_send_event(Pid,
{get_kvrange, StartKey, EndKey, ScanWidth}, {get_kvrange, StartKey, EndKey, ScanWidth},
infinity) of infinity) of
{yield, SlotsToFetchBinList, SlotsToPoint} -> {yield, SlotsToFetchBinList, SlotsToPoint, PressMethod} ->
FetchFun = FetchFun =
fun({SlotBin, SK, EK}, Acc) -> fun({SlotBin, SK, EK}, Acc) ->
Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK) Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK, PressMethod)
end, end,
lists:foldl(FetchFun, [], SlotsToFetchBinList) ++ SlotsToPoint; lists:foldl(FetchFun, [], SlotsToFetchBinList) ++ SlotsToPoint;
Reply -> Reply ->
@ -276,10 +287,11 @@ sst_getkvrange(Pid, StartKey, EndKey, ScanWidth) ->
%% Get a list of slots by their ID. The slot will be converted from the binary %% Get a list of slots by their ID. The slot will be converted from the binary
%% to term form outside of the FSM loop %% to term form outside of the FSM loop
sst_getslots(Pid, SlotList) -> sst_getslots(Pid, SlotList) ->
SlotBins = gen_fsm:sync_send_event(Pid, {get_slots, SlotList}, infinity), {SlotBins, PressMethod}
= gen_fsm:sync_send_event(Pid, {get_slots, SlotList}, infinity),
FetchFun = FetchFun =
fun({SlotBin, SK, EK}, Acc) -> fun({SlotBin, SK, EK}, Acc) ->
Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK) Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK, PressMethod)
end, end,
lists:foldl(FetchFun, [], SlotBins). lists:foldl(FetchFun, [], SlotBins).
@ -350,19 +362,22 @@ starting({sst_open, RootPath, Filename}, _From, State) ->
{ok, {Summary#summary.first_key, Summary#summary.last_key}}, {ok, {Summary#summary.first_key, Summary#summary.last_key}},
reader, reader,
UpdState}; UpdState};
starting({sst_new, RootPath, Filename, Level, {SlotList, FirstKey}, MaxSQN}, starting({sst_new,
_From, State) -> RootPath, Filename, Level,
{SlotList, FirstKey}, MaxSQN,
PressMethod}, _From, State) ->
SW = os:timestamp(), SW = os:timestamp(),
{Length, {Length,
SlotIndex, SlotIndex,
BlockIndex, BlockIndex,
SlotsBin} = build_all_slots(SlotList), SlotsBin} = build_all_slots(SlotList, PressMethod),
SummaryBin = build_table_summary(SlotIndex, SummaryBin = build_table_summary(SlotIndex,
Level, Level,
FirstKey, FirstKey,
Length, Length,
MaxSQN), MaxSQN),
ActualFilename = write_file(RootPath, Filename, SummaryBin, SlotsBin), ActualFilename =
write_file(RootPath, Filename, SummaryBin, SlotsBin, PressMethod),
YBQ = Level =< 2, YBQ = Level =< 2,
UpdState = read_file(ActualFilename, UpdState = read_file(ActualFilename,
State#state{root_path=RootPath, State#state{root_path=RootPath,
@ -377,20 +392,22 @@ starting({sst_new, RootPath, Filename, Level, {SlotList, FirstKey}, MaxSQN},
UpdState#state{blockindex_cache = BlockIndex}}. UpdState#state{blockindex_cache = BlockIndex}}.
starting({sst_newlevelzero, RootPath, Filename, starting({sst_newlevelzero, RootPath, Filename,
Slots, FetchFun, Penciller, MaxSQN}, State) -> Slots, FetchFun, Penciller, MaxSQN,
PressMethod}, State) ->
SW = os:timestamp(), SW = os:timestamp(),
KVList = leveled_pmem:to_list(Slots, FetchFun), KVList = leveled_pmem:to_list(Slots, FetchFun),
{[], [], SlotList, FirstKey} = merge_lists(KVList), {[], [], SlotList, FirstKey} = merge_lists(KVList, PressMethod),
{SlotCount, {SlotCount,
SlotIndex, SlotIndex,
BlockIndex, BlockIndex,
SlotsBin} = build_all_slots(SlotList), SlotsBin} = build_all_slots(SlotList, PressMethod),
SummaryBin = build_table_summary(SlotIndex, SummaryBin = build_table_summary(SlotIndex,
0, 0,
FirstKey, FirstKey,
SlotCount, SlotCount,
MaxSQN), MaxSQN),
ActualFilename = write_file(RootPath, Filename, SummaryBin, SlotsBin), ActualFilename =
write_file(RootPath, Filename, SummaryBin, SlotsBin, PressMethod),
UpdState = read_file(ActualFilename, UpdState = read_file(ActualFilename,
State#state{root_path = RootPath, State#state{root_path = RootPath,
yield_blockquery = true}), yield_blockquery = true}),
@ -400,13 +417,17 @@ starting({sst_newlevelzero, RootPath, Filename,
SW), SW),
case Penciller of case Penciller of
undefined -> undefined ->
{next_state, reader, UpdState#state{blockindex_cache = BlockIndex}}; {next_state,
reader,
UpdState#state{blockindex_cache = BlockIndex}};
_ -> _ ->
leveled_penciller:pcl_confirml0complete(Penciller, leveled_penciller:pcl_confirml0complete(Penciller,
UpdState#state.filename, UpdState#state.filename,
Summary#summary.first_key, Summary#summary.first_key,
Summary#summary.last_key), Summary#summary.last_key),
{next_state, reader, UpdState#state{blockindex_cache = BlockIndex}} {next_state,
reader,
UpdState#state{blockindex_cache = BlockIndex}}
end. end.
@ -420,16 +441,20 @@ reader({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) ->
EndKey, EndKey,
ScanWidth, ScanWidth,
State), State),
PressMethod = State#state.compression_method,
case State#state.yield_blockquery of case State#state.yield_blockquery of
true -> true ->
{reply, {reply,
{yield, SlotsToFetchBinList, SlotsToPoint}, {yield,
SlotsToFetchBinList,
SlotsToPoint,
PressMethod},
reader, reader,
State}; State};
false -> false ->
FetchFun = FetchFun =
fun({SlotBin, SK, EK}, Acc) -> fun({SlotBin, SK, EK}, Acc) ->
Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK) Acc ++ binaryslot_trimmedlist(SlotBin, SK, EK, PressMethod)
end, end,
{reply, {reply,
lists:foldl(FetchFun, [], SlotsToFetchBinList) ++ SlotsToPoint, lists:foldl(FetchFun, [], SlotsToFetchBinList) ++ SlotsToPoint,
@ -438,7 +463,7 @@ reader({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) ->
end; end;
reader({get_slots, SlotList}, _From, State) -> reader({get_slots, SlotList}, _From, State) ->
SlotBins = read_slots(State#state.handle, SlotList), SlotBins = read_slots(State#state.handle, SlotList),
{reply, SlotBins, reader, State}; {reply, {SlotBins, State#state.compression_method}, reader, State};
reader(get_maxsequencenumber, _From, State) -> reader(get_maxsequencenumber, _From, State) ->
Summary = State#state.summary, Summary = State#state.summary,
{reply, Summary#summary.max_sqn, reader, State}; {reply, Summary#summary.max_sqn, reader, State};
@ -475,14 +500,19 @@ delete_pending({get_kvrange, StartKey, EndKey, ScanWidth}, _From, State) ->
ScanWidth, ScanWidth,
State), State),
% Always yield as about to clear and de-reference % Always yield as about to clear and de-reference
PressMethod = State#state.compression_method,
{reply, {reply,
{yield, SlotsToFetchBinList, SlotsToPoint}, {yield, SlotsToFetchBinList, SlotsToPoint, PressMethod},
delete_pending, delete_pending,
State, State,
?DELETE_TIMEOUT}; ?DELETE_TIMEOUT};
delete_pending({get_slots, SlotList}, _From, State) -> delete_pending({get_slots, SlotList}, _From, State) ->
SlotBins = read_slots(State#state.handle, SlotList), SlotBins = read_slots(State#state.handle, SlotList),
{reply, SlotBins, delete_pending, State, ?DELETE_TIMEOUT}; {reply,
{SlotBins, State#state.compression_method},
delete_pending,
State,
?DELETE_TIMEOUT};
delete_pending(close, _From, State) -> delete_pending(close, _From, State) ->
leveled_log:log("SST07", [State#state.filename]), leveled_log:log("SST07", [State#state.filename]),
ok = file:close(State#state.handle), ok = file:close(State#state.handle),
@ -528,6 +558,7 @@ code_change(_OldVsn, StateName, State, _Extra) ->
fetch(LedgerKey, Hash, State) -> fetch(LedgerKey, Hash, State) ->
Summary = State#state.summary, Summary = State#state.summary,
PressMethod = State#state.compression_method,
Slot = lookup_slot(LedgerKey, Summary#summary.index), Slot = lookup_slot(LedgerKey, Summary#summary.index),
SlotID = Slot#slot_index_value.slot_id, SlotID = Slot#slot_index_value.slot_id,
Bloom = Slot#slot_index_value.bloom, Bloom = Slot#slot_index_value.bloom,
@ -540,9 +571,8 @@ fetch(LedgerKey, Hash, State) ->
case CachedBlockIdx of case CachedBlockIdx of
none -> none ->
SlotBin = read_slot(State#state.handle, Slot), SlotBin = read_slot(State#state.handle, Slot),
{Result, {Result, BlockLengths, BlockIdx} =
BlockLengths, binaryslot_get(SlotBin, LedgerKey, Hash, PressMethod),
BlockIdx} = binaryslot_get(SlotBin, LedgerKey, Hash),
BlockIndexCache = array:set(SlotID - 1, BlockIndexCache = array:set(SlotID - 1,
<<BlockLengths/binary, <<BlockLengths/binary,
BlockIdx/binary>>, BlockIdx/binary>>,
@ -560,11 +590,13 @@ fetch(LedgerKey, Hash, State) ->
[] -> [] ->
{not_present, slot_bloom, SlotID, State}; {not_present, slot_bloom, SlotID, State};
_ -> _ ->
Result = check_blocks(PosList, Result =
check_blocks(PosList,
State#state.handle, State#state.handle,
Slot, Slot,
BlockLengths, BlockLengths,
LedgerKey), LedgerKey,
PressMethod),
{Result, slot_fetch, SlotID, State} {Result, slot_fetch, SlotID, State}
end end
end end
@ -626,12 +658,14 @@ fetch_range(StartKey, EndKey, ScanWidth, State) ->
{SlotsToFetchBinList, SlotsToPoint}. {SlotsToFetchBinList, SlotsToPoint}.
write_file(RootPath, Filename, SummaryBin, SlotsBin) -> write_file(RootPath, Filename, SummaryBin, SlotsBin, PressMethod) ->
SummaryLength = byte_size(SummaryBin), SummaryLength = byte_size(SummaryBin),
SlotsLength = byte_size(SlotsBin), SlotsLength = byte_size(SlotsBin),
{PendingName, FinalName} = generate_filenames(Filename), {PendingName, FinalName} = generate_filenames(Filename),
FileVersion = gen_fileversion(PressMethod),
ok = file:write_file(filename:join(RootPath, PendingName), ok = file:write_file(filename:join(RootPath, PendingName),
<<SlotsLength:32/integer, <<FileVersion:8/integer,
SlotsLength:32/integer,
SummaryLength:32/integer, SummaryLength:32/integer,
SlotsBin/binary, SlotsBin/binary,
SummaryBin/binary>>, SummaryBin/binary>>,
@ -650,27 +684,48 @@ write_file(RootPath, Filename, SummaryBin, SlotsBin) ->
FinalName. FinalName.
read_file(Filename, State) -> read_file(Filename, State) ->
{Handle, SummaryBin} = open_reader(filename:join(State#state.root_path, {Handle, FileVersion, SummaryBin} =
Filename)), open_reader(filename:join(State#state.root_path, Filename)),
UpdState0 = imp_fileversion(FileVersion, State),
{Summary, SlotList} = read_table_summary(SummaryBin), {Summary, SlotList} = read_table_summary(SummaryBin),
BlockIndexCache = array:new([{size, Summary#summary.size}, BlockIndexCache = array:new([{size, Summary#summary.size},
{default, none}]), {default, none}]),
UpdState = State#state{blockindex_cache = BlockIndexCache}, UpdState1 = UpdState0#state{blockindex_cache = BlockIndexCache},
SlotIndex = from_list(SlotList), SlotIndex = from_list(SlotList),
UpdSummary = Summary#summary{index = SlotIndex}, UpdSummary = Summary#summary{index = SlotIndex},
leveled_log:log("SST03", [Filename, leveled_log:log("SST03", [Filename,
Summary#summary.size, Summary#summary.size,
Summary#summary.max_sqn]), Summary#summary.max_sqn]),
UpdState#state{summary = UpdSummary, UpdState1#state{summary = UpdSummary,
handle = Handle, handle = Handle,
filename = Filename}. filename = Filename}.
gen_fileversion(PressMethod) ->
Bit1 =
case PressMethod of
lz4 -> 1;
native -> 0
end,
Bit1.
imp_fileversion(VersionInt, State) ->
UpdState =
case VersionInt band 1 of
0 ->
State#state{compression_method = native};
1 ->
State#state{compression_method = lz4}
end,
UpdState.
open_reader(Filename) -> open_reader(Filename) ->
{ok, Handle} = file:open(Filename, [binary, raw, read]), {ok, Handle} = file:open(Filename, [binary, raw, read]),
{ok, Lengths} = file:pread(Handle, 0, 8), {ok, Lengths} = file:pread(Handle, 0, 9),
<<SlotsLength:32/integer, SummaryLength:32/integer>> = Lengths, <<FileVersion:8/integer,
{ok, SummaryBin} = file:pread(Handle, SlotsLength + 8, SummaryLength), SlotsLength:32/integer,
{Handle, SummaryBin}. SummaryLength:32/integer>> = Lengths,
{ok, SummaryBin} = file:pread(Handle, SlotsLength + 9, SummaryLength),
{Handle, FileVersion, SummaryBin}.
build_table_summary(SlotIndex, _Level, FirstKey, SlotCount, MaxSQN) -> build_table_summary(SlotIndex, _Level, FirstKey, SlotCount, MaxSQN) ->
[{LastKey, _LastV}|_Rest] = SlotIndex, [{LastKey, _LastV}|_Rest] = SlotIndex,
@ -693,23 +748,26 @@ read_table_summary(BinWithCheck) ->
end. end.
build_all_slots(SlotList) -> build_all_slots(SlotList, PressMethod) ->
SlotCount = length(SlotList), SlotCount = length(SlotList),
BuildResponse = build_all_slots(SlotList, BuildResponse = build_all_slots(SlotList,
8, 9,
1, 1,
[], [],
array:new([{size, SlotCount}, array:new([{size, SlotCount},
{default, none}]), {default, none}]),
<<>>), <<>>,
PressMethod),
{SlotIndex, BlockIndex, SlotsBin} = BuildResponse, {SlotIndex, BlockIndex, SlotsBin} = BuildResponse,
{SlotCount, SlotIndex, BlockIndex, SlotsBin}. {SlotCount, SlotIndex, BlockIndex, SlotsBin}.
build_all_slots([], _Pos, _SlotID, build_all_slots([], _Pos, _SlotID,
SlotIdxAcc, BlockIdxAcc, SlotBinAcc) -> SlotIdxAcc, BlockIdxAcc, SlotBinAcc,
_PressMethod) ->
{SlotIdxAcc, BlockIdxAcc, SlotBinAcc}; {SlotIdxAcc, BlockIdxAcc, SlotBinAcc};
build_all_slots([SlotD|Rest], Pos, SlotID, build_all_slots([SlotD|Rest], Pos, SlotID,
SlotIdxAcc, BlockIdxAcc, SlotBinAcc) -> SlotIdxAcc, BlockIdxAcc, SlotBinAcc,
PressMethod) ->
{BlockIdx, SlotBin, HashList, LastKey} = SlotD, {BlockIdx, SlotBin, HashList, LastKey} = SlotD,
Length = byte_size(SlotBin), Length = byte_size(SlotBin),
Bloom = leveled_tinybloom:create_bloom(HashList), Bloom = leveled_tinybloom:create_bloom(HashList),
@ -722,7 +780,8 @@ build_all_slots([SlotD|Rest], Pos, SlotID,
SlotID + 1, SlotID + 1,
[{LastKey, SlotIndexV}|SlotIdxAcc], [{LastKey, SlotIndexV}|SlotIdxAcc],
array:set(SlotID - 1, BlockIdx, BlockIdxAcc), array:set(SlotID - 1, BlockIdx, BlockIdxAcc),
<<SlotBinAcc/binary, SlotBin/binary>>). <<SlotBinAcc/binary, SlotBin/binary>>,
PressMethod).
generate_filenames(RootFilename) -> generate_filenames(RootFilename) ->
@ -740,26 +799,30 @@ generate_filenames(RootFilename) ->
end. end.
-spec serialise_block(any()) -> binary(). -spec serialise_block(any(), lz4|native) -> binary().
%% @doc %% @doc
%% Convert term to binary %% Convert term to binary
%% Function split out to make it easier to experiment with different %% Function split out to make it easier to experiment with different
%% compression methods. Also, perhaps standardise applictaion of CRC %% compression methods. Also, perhaps standardise applictaion of CRC
%% checks %% checks
serialise_block(Term) -> serialise_block(Term, lz4) ->
{ok, Bin} = lz4:pack(term_to_binary(Term)), {ok, Bin} = lz4:pack(term_to_binary(Term)),
Bin. Bin;
serialise_block(Term, native) ->
term_to_binary(Term, ?BINARY_SETTINGS).
-spec deserialise_block(binary()) -> any(). -spec deserialise_block(binary(), lz4|native) -> any().
%% @doc %% @doc
%% Convert binary to term %% Convert binary to term
%% Function split out to make it easier to experiment with different %% Function split out to make it easier to experiment with different
%% compression methods. Also, perhaps standardise applictaion of CRC %% compression methods. Also, perhaps standardise applictaion of CRC
%% checks %% checks
deserialise_block(Bin) -> deserialise_block(Bin, lz4) ->
{ok, Bin0} = lz4:unpack(Bin), {ok, Bin0} = lz4:unpack(Bin),
binary_to_term(Bin0). binary_to_term(Bin0);
deserialise_block(Bin, native) ->
binary_to_term(Bin).
%%%============================================================================ %%%============================================================================
@ -823,7 +886,7 @@ lookup_slots(StartKey, EndKey, Tree) ->
%% based on a 17-bit hash (so 0.0039 fpr). %% based on a 17-bit hash (so 0.0039 fpr).
generate_binary_slot(Lookup, KVL) -> generate_binary_slot(Lookup, KVL, PressMethod) ->
HashFoldFun = HashFoldFun =
fun({K, V}, {PosBinAcc, NoHashCount, HashAcc}) -> fun({K, V}, {PosBinAcc, NoHashCount, HashAcc}) ->
@ -887,45 +950,45 @@ generate_binary_slot(Lookup, KVL) ->
{B1, B2, B3, B4, B5} = {B1, B2, B3, B4, B5} =
case length(KVL) of case length(KVL) of
L when L =< SideBlockSize -> L when L =< SideBlockSize ->
{serialise_block(KVL), {serialise_block(KVL, PressMethod),
<<0:0>>, <<0:0>>,
<<0:0>>, <<0:0>>,
<<0:0>>, <<0:0>>,
<<0:0>>}; <<0:0>>};
L when L =< 2 * SideBlockSize -> L when L =< 2 * SideBlockSize ->
{KVLA, KVLB} = lists:split(SideBlockSize, KVL), {KVLA, KVLB} = lists:split(SideBlockSize, KVL),
{serialise_block(KVLA), {serialise_block(KVLA, PressMethod),
serialise_block(KVLB), serialise_block(KVLB, PressMethod),
<<0:0>>, <<0:0>>,
<<0:0>>, <<0:0>>,
<<0:0>>}; <<0:0>>};
L when L =< (2 * SideBlockSize + MidBlockSize) -> L when L =< (2 * SideBlockSize + MidBlockSize) ->
{KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL), {KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL),
{KVLB, KVLC} = lists:split(SideBlockSize, KVLB_Rest), {KVLB, KVLC} = lists:split(SideBlockSize, KVLB_Rest),
{serialise_block(KVLA), {serialise_block(KVLA, PressMethod),
serialise_block(KVLB), serialise_block(KVLB, PressMethod),
serialise_block(KVLC), serialise_block(KVLC, PressMethod),
<<0:0>>, <<0:0>>,
<<0:0>>}; <<0:0>>};
L when L =< (3 * SideBlockSize + MidBlockSize) -> L when L =< (3 * SideBlockSize + MidBlockSize) ->
{KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL), {KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL),
{KVLB, KVLC_Rest} = lists:split(SideBlockSize, KVLB_Rest), {KVLB, KVLC_Rest} = lists:split(SideBlockSize, KVLB_Rest),
{KVLC, KVLD} = lists:split(MidBlockSize, KVLC_Rest), {KVLC, KVLD} = lists:split(MidBlockSize, KVLC_Rest),
{serialise_block(KVLA), {serialise_block(KVLA, PressMethod),
serialise_block(KVLB), serialise_block(KVLB, PressMethod),
serialise_block(KVLC), serialise_block(KVLC, PressMethod),
serialise_block(KVLD), serialise_block(KVLD, PressMethod),
<<0:0>>}; <<0:0>>};
L when L =< (4 * SideBlockSize + MidBlockSize) -> L when L =< (4 * SideBlockSize + MidBlockSize) ->
{KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL), {KVLA, KVLB_Rest} = lists:split(SideBlockSize, KVL),
{KVLB, KVLC_Rest} = lists:split(SideBlockSize, KVLB_Rest), {KVLB, KVLC_Rest} = lists:split(SideBlockSize, KVLB_Rest),
{KVLC, KVLD_Rest} = lists:split(MidBlockSize, KVLC_Rest), {KVLC, KVLD_Rest} = lists:split(MidBlockSize, KVLC_Rest),
{KVLD, KVLE} = lists:split(SideBlockSize, KVLD_Rest), {KVLD, KVLE} = lists:split(SideBlockSize, KVLD_Rest),
{serialise_block(KVLA), {serialise_block(KVLA, PressMethod),
serialise_block(KVLB), serialise_block(KVLB, PressMethod),
serialise_block(KVLC), serialise_block(KVLC, PressMethod),
serialise_block(KVLD), serialise_block(KVLD, PressMethod),
serialise_block(KVLE)} serialise_block(KVLE, PressMethod)}
end, end,
B1P = byte_size(PosBinIndex), B1P = byte_size(PosBinIndex),
@ -951,18 +1014,21 @@ generate_binary_slot(Lookup, KVL) ->
{<<Lengths/binary, PosBinIndex/binary>>, FullBin, HashL, LastKey}. {<<Lengths/binary, PosBinIndex/binary>>, FullBin, HashL, LastKey}.
check_blocks([], _Handle, _Slot, _BlockLengths, _LedgerKey) -> check_blocks([], _Handle, _Slot, _BlockLengths, _LedgerKey, _PressMethod) ->
not_present; not_present;
check_blocks([Pos|Rest], Handle, Slot, BlockLengths, LedgerKey) -> check_blocks([Pos|Rest], Handle, Slot, BlockLengths, LedgerKey, PressMethod) ->
{BlockNumber, BlockPos} = revert_position(Pos), {BlockNumber, BlockPos} = revert_position(Pos),
BlockBin = read_block(Handle, Slot, BlockLengths, BlockNumber), BlockBin = read_block(Handle, Slot, BlockLengths, BlockNumber),
BlockL = deserialise_block(BlockBin), BlockL = deserialise_block(BlockBin, PressMethod),
{K, V} = lists:nth(BlockPos, BlockL), {K, V} = lists:nth(BlockPos, BlockL),
case K of case K of
LedgerKey -> LedgerKey ->
{K, V}; {K, V};
_ -> _ ->
check_blocks(Rest, Handle, Slot, BlockLengths, LedgerKey) check_blocks(Rest,
Handle, Slot, BlockLengths,
LedgerKey,
PressMethod)
end. end.
@ -1018,7 +1084,7 @@ read_slots(Handle, SlotList) ->
lists:map(BinSplitMapFun, LengthList). lists:map(BinSplitMapFun, LengthList).
binaryslot_get(FullBin, Key, Hash) -> binaryslot_get(FullBin, Key, Hash, PressMethod) ->
case crc_check_slot(FullBin) of case crc_check_slot(FullBin) of
{BlockLengths, Rest} -> {BlockLengths, Rest} ->
<<B1P:32/integer, _R/binary>> = BlockLengths, <<B1P:32/integer, _R/binary>> = BlockLengths,
@ -1027,7 +1093,7 @@ binaryslot_get(FullBin, Key, Hash) ->
extra_hash(Hash), extra_hash(Hash),
[], [],
0), 0),
{fetch_value(PosList, BlockLengths, Blocks, Key), {fetch_value(PosList, BlockLengths, Blocks, Key, PressMethod),
BlockLengths, BlockLengths,
PosBinIndex}; PosBinIndex};
crc_wonky -> crc_wonky ->
@ -1036,7 +1102,7 @@ binaryslot_get(FullBin, Key, Hash) ->
none} none}
end. end.
binaryslot_tolist(FullBin) -> binaryslot_tolist(FullBin, PressMethod) ->
BlockFetchFun = BlockFetchFun =
fun(Length, {Acc, Bin}) -> fun(Length, {Acc, Bin}) ->
case Length of case Length of
@ -1044,7 +1110,7 @@ binaryslot_tolist(FullBin) ->
{Acc, Bin}; {Acc, Bin};
_ -> _ ->
<<Block:Length/binary, Rest/binary>> = Bin, <<Block:Length/binary, Rest/binary>> = Bin,
{Acc ++ deserialise_block(Block), Rest} {Acc ++ deserialise_block(Block, PressMethod), Rest}
end end
end, end,
@ -1067,9 +1133,9 @@ binaryslot_tolist(FullBin) ->
Out. Out.
binaryslot_trimmedlist(FullBin, all, all) -> binaryslot_trimmedlist(FullBin, all, all, PressMethod) ->
binaryslot_tolist(FullBin); binaryslot_tolist(FullBin, PressMethod);
binaryslot_trimmedlist(FullBin, StartKey, EndKey) -> binaryslot_trimmedlist(FullBin, StartKey, EndKey, PressMethod) ->
LTrimFun = fun({K, _V}) -> K < StartKey end, LTrimFun = fun({K, _V}) -> K < StartKey end,
RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end, RTrimFun = fun({K, _V}) -> not leveled_codec:endkey_passed(EndKey, K) end,
@ -1098,7 +1164,8 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey) ->
0 -> 0 ->
[Block1, Block2]; [Block1, Block2];
_ -> _ ->
MidBlockList = deserialise_block(MidBlock), MidBlockList =
deserialise_block(MidBlock, PressMethod),
{MidFirst, _} = lists:nth(1, MidBlockList), {MidFirst, _} = lists:nth(1, MidBlockList),
{MidLast, _} = lists:last(MidBlockList), {MidLast, _} = lists:last(MidBlockList),
Split = {StartKey > MidLast, Split = {StartKey > MidLast,
@ -1136,7 +1203,7 @@ binaryslot_trimmedlist(FullBin, StartKey, EndKey) ->
BlockList = BlockList =
case is_binary(Block) of case is_binary(Block) of
true -> true ->
deserialise_block(Block); deserialise_block(Block, PressMethod);
false -> false ->
Block Block
end, end,
@ -1212,21 +1279,21 @@ extra_hash({SegHash, _ExtraHash}) when is_integer(SegHash) ->
extra_hash(NotHash) -> extra_hash(NotHash) ->
NotHash. NotHash.
fetch_value([], _BlockLengths, _Blocks, _Key) -> fetch_value([], _BlockLengths, _Blocks, _Key, _PressMethod) ->
not_present; not_present;
fetch_value([Pos|Rest], BlockLengths, Blocks, Key) -> fetch_value([Pos|Rest], BlockLengths, Blocks, Key, PressMethod) ->
{BlockNumber, BlockPos} = revert_position(Pos), {BlockNumber, BlockPos} = revert_position(Pos),
{_BlockPos, {_BlockPos,
Offset, Offset,
Length} = block_offsetandlength(BlockLengths, BlockNumber), Length} = block_offsetandlength(BlockLengths, BlockNumber),
<<_Pre:Offset/binary, Block:Length/binary, _Rest/binary>> = Blocks, <<_Pre:Offset/binary, Block:Length/binary, _Rest/binary>> = Blocks,
BlockL = deserialise_block(Block), BlockL = deserialise_block(Block, PressMethod),
{K, V} = lists:nth(BlockPos, BlockL), {K, V} = lists:nth(BlockPos, BlockL),
case K of case K of
Key -> Key ->
{K, V}; {K, V};
_ -> _ ->
fetch_value(Rest, BlockLengths, Blocks, Key) fetch_value(Rest, BlockLengths, Blocks, Key, PressMethod)
end. end.
@ -1290,31 +1357,33 @@ find_pos(<<0:1/integer, NHC:7/integer, T/binary>>, Hash, PosList, Count) ->
%% there are matching keys then the highest sequence number must be chosen and %% there are matching keys then the highest sequence number must be chosen and
%% any lower sequence numbers should be compacted out of existence %% any lower sequence numbers should be compacted out of existence
merge_lists(KVList1) -> merge_lists(KVList1, PressMethod) ->
SlotCount = length(KVList1) div ?LOOK_SLOTSIZE, SlotCount = length(KVList1) div ?LOOK_SLOTSIZE,
{[], {[],
[], [],
split_lists(KVList1, [], SlotCount), split_lists(KVList1, [], SlotCount, PressMethod),
element(1, lists:nth(1, KVList1))}. element(1, lists:nth(1, KVList1))}.
split_lists([], SlotLists, 0) -> split_lists([], SlotLists, 0, _PressMethod) ->
lists:reverse(SlotLists); lists:reverse(SlotLists);
split_lists(LastPuff, SlotLists, 0) -> split_lists(LastPuff, SlotLists, 0, PressMethod) ->
SlotD = generate_binary_slot(lookup, LastPuff), SlotD = generate_binary_slot(lookup, LastPuff, PressMethod),
lists:reverse([SlotD|SlotLists]); lists:reverse([SlotD|SlotLists]);
split_lists(KVList1, SlotLists, N) -> split_lists(KVList1, SlotLists, N, PressMethod) ->
{Slot, KVListRem} = lists:split(?LOOK_SLOTSIZE, KVList1), {Slot, KVListRem} = lists:split(?LOOK_SLOTSIZE, KVList1),
SlotD = generate_binary_slot(lookup, Slot), SlotD = generate_binary_slot(lookup, Slot, PressMethod),
split_lists(KVListRem, [SlotD|SlotLists], N - 1). split_lists(KVListRem, [SlotD|SlotLists], N - 1, PressMethod).
merge_lists(KVList1, KVList2, LevelInfo) -> merge_lists(KVList1, KVList2, LevelInfo, PressMethod) ->
merge_lists(KVList1, KVList2, LevelInfo, [], null, 0). merge_lists(KVList1, KVList2, LevelInfo, [], null, 0, PressMethod).
merge_lists(KVList1, KVList2, _LI, SlotList, FirstKey, ?MAX_SLOTS) -> merge_lists(KVList1, KVList2, _LI, SlotList, FirstKey, ?MAX_SLOTS,
_PressMethod) ->
{KVList1, KVList2, lists:reverse(SlotList), FirstKey}; {KVList1, KVList2, lists:reverse(SlotList), FirstKey};
merge_lists([], [], _LI, SlotList, FirstKey, _SlotCount) -> merge_lists([], [], _LI, SlotList, FirstKey, _SlotCount, _PressMethod) ->
{[], [], lists:reverse(SlotList), FirstKey}; {[], [], lists:reverse(SlotList), FirstKey};
merge_lists(KVList1, KVList2, LI, SlotList, FirstKey, SlotCount) -> merge_lists(KVList1, KVList2, LI, SlotList, FirstKey, SlotCount,
PressMethod) ->
{KVRem1, KVRem2, Slot, FK0} = {KVRem1, KVRem2, Slot, FK0} =
form_slot(KVList1, KVList2, LI, no_lookup, 0, [], FirstKey), form_slot(KVList1, KVList2, LI, no_lookup, 0, [], FirstKey),
case Slot of case Slot of
@ -1324,15 +1393,17 @@ merge_lists(KVList1, KVList2, LI, SlotList, FirstKey, SlotCount) ->
LI, LI,
SlotList, SlotList,
FK0, FK0,
SlotCount); SlotCount,
PressMethod);
{Lookup, KVL} -> {Lookup, KVL} ->
SlotD = generate_binary_slot(Lookup, KVL), SlotD = generate_binary_slot(Lookup, KVL, PressMethod),
merge_lists(KVRem1, merge_lists(KVRem1,
KVRem2, KVRem2,
LI, LI,
[SlotD|SlotList], [SlotD|SlotList],
FK0, FK0,
SlotCount + 1) SlotCount + 1,
PressMethod)
end. end.
form_slot([], [], _LI, Type, _Size, Slot, FK) -> form_slot([], [], _LI, Type, _Size, Slot, FK) ->
@ -1545,7 +1616,7 @@ form_slot_test() ->
?assertMatch({[], [], {no_lookup, Slot}, {o, "B1", "K5", null}}, R1). ?assertMatch({[], [], {no_lookup, Slot}, {o, "B1", "K5", null}}, R1).
merge_tombstonelist_test() -> merge_tombstonelist_test() ->
% Merge lists wiht nothing but tombstones % Merge lists with nothing but tombstones
SkippingKV1 = {{o, "B1", "K9995", null}, {9995, tomb, 1234567, {}}}, SkippingKV1 = {{o, "B1", "K9995", null}, {9995, tomb, 1234567, {}}},
SkippingKV2 = {{o, "B1", "K9996", null}, {9996, tomb, 1234567, {}}}, SkippingKV2 = {{o, "B1", "K9996", null}, {9996, tomb, 1234567, {}}},
SkippingKV3 = {{o, "B1", "K9997", null}, {9997, tomb, 1234567, {}}}, SkippingKV3 = {{o, "B1", "K9997", null}, {9997, tomb, 1234567, {}}},
@ -1553,7 +1624,8 @@ merge_tombstonelist_test() ->
SkippingKV5 = {{o, "B1", "K9999", null}, {9999, tomb, 1234567, {}}}, SkippingKV5 = {{o, "B1", "K9999", null}, {9999, tomb, 1234567, {}}},
R = merge_lists([SkippingKV1, SkippingKV3, SkippingKV5], R = merge_lists([SkippingKV1, SkippingKV3, SkippingKV5],
[SkippingKV2, SkippingKV4], [SkippingKV2, SkippingKV4],
{true, 9999999}), {true, 9999999},
native),
?assertMatch({[], [], [], null}, R). ?assertMatch({[], [], [], null}, R).
indexed_list_test() -> indexed_list_test() ->
@ -1564,7 +1636,8 @@ indexed_list_test() ->
SW0 = os:timestamp(), SW0 = os:timestamp(),
{_PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(lookup, KVL1), {_PosBinIndex1, FullBin, _HL, _LK} =
generate_binary_slot(lookup, KVL1, native),
io:format(user, io:format(user,
"Indexed list created slot in ~w microseconds of size ~w~n", "Indexed list created slot in ~w microseconds of size ~w~n",
[timer:now_diff(os:timestamp(), SW0), byte_size(FullBin)]), [timer:now_diff(os:timestamp(), SW0), byte_size(FullBin)]),
@ -1592,7 +1665,8 @@ indexed_list_mixedkeys_test() ->
KVL1 = lists:sublist(KVL0, 33), KVL1 = lists:sublist(KVL0, 33),
Keys = lists:ukeysort(1, generate_indexkeys(60) ++ KVL1), Keys = lists:ukeysort(1, generate_indexkeys(60) ++ KVL1),
{_PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(lookup, Keys), {_PosBinIndex1, FullBin, _HL, _LK} =
generate_binary_slot(lookup, Keys, native),
{TestK1, TestV1} = lists:nth(4, KVL1), {TestK1, TestV1} = lists:nth(4, KVL1),
MH1 = leveled_codec:segment_hash(TestK1), MH1 = leveled_codec:segment_hash(TestK1),
@ -1618,7 +1692,8 @@ indexed_list_mixedkeys2_test() ->
IdxKeys2 = lists:ukeysort(1, generate_indexkeys(30)), IdxKeys2 = lists:ukeysort(1, generate_indexkeys(30)),
% this isn't actually ordered correctly % this isn't actually ordered correctly
Keys = IdxKeys1 ++ KVL1 ++ IdxKeys2, Keys = IdxKeys1 ++ KVL1 ++ IdxKeys2,
{_PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(lookup, Keys), {_PosBinIndex1, FullBin, _HL, _LK} =
generate_binary_slot(lookup, Keys, native),
lists:foreach(fun({K, V}) -> lists:foreach(fun({K, V}) ->
MH = leveled_codec:segment_hash(K), MH = leveled_codec:segment_hash(K),
test_binary_slot(FullBin, K, MH, {K, V}) test_binary_slot(FullBin, K, MH, {K, V})
@ -1628,34 +1703,37 @@ indexed_list_mixedkeys2_test() ->
indexed_list_allindexkeys_test() -> indexed_list_allindexkeys_test() ->
Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)),
?LOOK_SLOTSIZE), ?LOOK_SLOTSIZE),
{PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(lookup, Keys), {PosBinIndex1, FullBin, _HL, _LK} =
generate_binary_slot(lookup, Keys, native),
EmptySlotSize = ?LOOK_SLOTSIZE - 1, EmptySlotSize = ?LOOK_SLOTSIZE - 1,
?assertMatch(<<_BL:24/binary, EmptySlotSize:8/integer>>, PosBinIndex1), ?assertMatch(<<_BL:24/binary, EmptySlotSize:8/integer>>, PosBinIndex1),
% SW = os:timestamp(), % SW = os:timestamp(),
BinToList = binaryslot_tolist(FullBin), BinToList = binaryslot_tolist(FullBin, native),
% io:format(user, % io:format(user,
% "Indexed list flattened in ~w microseconds ~n", % "Indexed list flattened in ~w microseconds ~n",
% [timer:now_diff(os:timestamp(), SW)]), % [timer:now_diff(os:timestamp(), SW)]),
?assertMatch(Keys, BinToList), ?assertMatch(Keys, BinToList),
?assertMatch(Keys, binaryslot_trimmedlist(FullBin, all, all)). ?assertMatch(Keys, binaryslot_trimmedlist(FullBin, all, all, native)).
indexed_list_allindexkeys_nolookup_test() -> indexed_list_allindexkeys_nolookup_test() ->
Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(1000)), Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(1000)),
?NOLOOK_SLOTSIZE), ?NOLOOK_SLOTSIZE),
{PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(no_lookup, Keys), {PosBinIndex1, FullBin, _HL, _LK} =
generate_binary_slot(no_lookup, Keys, native),
?assertMatch(<<_BL:24/binary, 127:8/integer>>, PosBinIndex1), ?assertMatch(<<_BL:24/binary, 127:8/integer>>, PosBinIndex1),
% SW = os:timestamp(), % SW = os:timestamp(),
BinToList = binaryslot_tolist(FullBin), BinToList = binaryslot_tolist(FullBin, native),
% io:format(user, % io:format(user,
% "Indexed list flattened in ~w microseconds ~n", % "Indexed list flattened in ~w microseconds ~n",
% [timer:now_diff(os:timestamp(), SW)]), % [timer:now_diff(os:timestamp(), SW)]),
?assertMatch(Keys, BinToList), ?assertMatch(Keys, BinToList),
?assertMatch(Keys, binaryslot_trimmedlist(FullBin, all, all)). ?assertMatch(Keys, binaryslot_trimmedlist(FullBin, all, all, native)).
indexed_list_allindexkeys_trimmed_test() -> indexed_list_allindexkeys_trimmed_test() ->
Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)), Keys = lists:sublist(lists:ukeysort(1, generate_indexkeys(150)),
?LOOK_SLOTSIZE), ?LOOK_SLOTSIZE),
{PosBinIndex1, FullBin, _HL, _LK} = generate_binary_slot(lookup, Keys), {PosBinIndex1, FullBin, _HL, _LK} =
generate_binary_slot(lookup, Keys, native),
EmptySlotSize = ?LOOK_SLOTSIZE - 1, EmptySlotSize = ?LOOK_SLOTSIZE - 1,
?assertMatch(<<_BL:24/binary, EmptySlotSize:8/integer>>, PosBinIndex1), ?assertMatch(<<_BL:24/binary, EmptySlotSize:8/integer>>, PosBinIndex1),
?assertMatch(Keys, binaryslot_trimmedlist(FullBin, ?assertMatch(Keys, binaryslot_trimmedlist(FullBin,
@ -1666,26 +1744,27 @@ indexed_list_allindexkeys_trimmed_test() ->
{i, {i,
"Bucket", "Bucket",
{"t1_int", 99999}, {"t1_int", 99999},
null})), null},
native)),
{SK1, _} = lists:nth(10, Keys), {SK1, _} = lists:nth(10, Keys),
{EK1, _} = lists:nth(100, Keys), {EK1, _} = lists:nth(100, Keys),
R1 = lists:sublist(Keys, 10, 91), R1 = lists:sublist(Keys, 10, 91),
O1 = binaryslot_trimmedlist(FullBin, SK1, EK1), O1 = binaryslot_trimmedlist(FullBin, SK1, EK1, native),
?assertMatch(91, length(O1)), ?assertMatch(91, length(O1)),
?assertMatch(R1, O1), ?assertMatch(R1, O1),
{SK2, _} = lists:nth(10, Keys), {SK2, _} = lists:nth(10, Keys),
{EK2, _} = lists:nth(20, Keys), {EK2, _} = lists:nth(20, Keys),
R2 = lists:sublist(Keys, 10, 11), R2 = lists:sublist(Keys, 10, 11),
O2 = binaryslot_trimmedlist(FullBin, SK2, EK2), O2 = binaryslot_trimmedlist(FullBin, SK2, EK2, native),
?assertMatch(11, length(O2)), ?assertMatch(11, length(O2)),
?assertMatch(R2, O2), ?assertMatch(R2, O2),
{SK3, _} = lists:nth(?LOOK_SLOTSIZE - 1, Keys), {SK3, _} = lists:nth(?LOOK_SLOTSIZE - 1, Keys),
{EK3, _} = lists:nth(?LOOK_SLOTSIZE, Keys), {EK3, _} = lists:nth(?LOOK_SLOTSIZE, Keys),
R3 = lists:sublist(Keys, ?LOOK_SLOTSIZE - 1, 2), R3 = lists:sublist(Keys, ?LOOK_SLOTSIZE - 1, 2),
O3 = binaryslot_trimmedlist(FullBin, SK3, EK3), O3 = binaryslot_trimmedlist(FullBin, SK3, EK3, native),
?assertMatch(2, length(O3)), ?assertMatch(2, length(O3)),
?assertMatch(R3, O3). ?assertMatch(R3, O3).
@ -1694,7 +1773,8 @@ indexed_list_mixedkeys_bitflip_test() ->
KVL0 = lists:ukeysort(1, generate_randomkeys(1, 50, 1, 4)), KVL0 = lists:ukeysort(1, generate_randomkeys(1, 50, 1, 4)),
KVL1 = lists:sublist(KVL0, 33), KVL1 = lists:sublist(KVL0, 33),
Keys = lists:ukeysort(1, generate_indexkeys(60) ++ KVL1), Keys = lists:ukeysort(1, generate_indexkeys(60) ++ KVL1),
{_PosBinIndex1, FullBin, _HL, LK} = generate_binary_slot(lookup, Keys), {_PosBinIndex1, FullBin, _HL, LK} =
generate_binary_slot(lookup, Keys, native),
?assertMatch(LK, element(1, lists:last(Keys))), ?assertMatch(LK, element(1, lists:last(Keys))),
L = byte_size(FullBin), L = byte_size(FullBin),
Byte1 = leveled_rand:uniform(L), Byte1 = leveled_rand:uniform(L),
@ -1711,12 +1791,12 @@ indexed_list_mixedkeys_bitflip_test() ->
MH1 = leveled_codec:segment_hash(TestK1), MH1 = leveled_codec:segment_hash(TestK1),
test_binary_slot(FullBin0, TestK1, MH1, not_present), test_binary_slot(FullBin0, TestK1, MH1, not_present),
ToList = binaryslot_tolist(FullBin0), ToList = binaryslot_tolist(FullBin0, native),
?assertMatch([], ToList), ?assertMatch([], ToList),
{SK1, _} = lists:nth(10, Keys), {SK1, _} = lists:nth(10, Keys),
{EK1, _} = lists:nth(50, Keys), {EK1, _} = lists:nth(50, Keys),
O1 = binaryslot_trimmedlist(FullBin0, SK1, EK1), O1 = binaryslot_trimmedlist(FullBin0, SK1, EK1, native),
?assertMatch(0, length(O1)), ?assertMatch(0, length(O1)),
?assertMatch([], O1). ?assertMatch([], O1).
@ -1724,7 +1804,7 @@ indexed_list_mixedkeys_bitflip_test() ->
test_binary_slot(FullBin, Key, Hash, ExpectedValue) -> test_binary_slot(FullBin, Key, Hash, ExpectedValue) ->
% SW = os:timestamp(), % SW = os:timestamp(),
{ReturnedValue, _BLs, _Idx} = binaryslot_get(FullBin, Key, Hash), {ReturnedValue, _BLs, _Idx} = binaryslot_get(FullBin, Key, Hash, native),
?assertMatch(ExpectedValue, ReturnedValue). ?assertMatch(ExpectedValue, ReturnedValue).
% io:format(user, "Fetch success in ~w microseconds ~n", % io:format(user, "Fetch success in ~w microseconds ~n",
% [timer:now_diff(os:timestamp(), SW)]). % [timer:now_diff(os:timestamp(), SW)]).
@ -1737,8 +1817,10 @@ merge_test() ->
KVL2 = lists:ukeysort(1, generate_randomkeys(1, N, 1, 20)), KVL2 = lists:ukeysort(1, generate_randomkeys(1, N, 1, 20)),
KVL3 = lists:ukeymerge(1, KVL1, KVL2), KVL3 = lists:ukeymerge(1, KVL1, KVL2),
SW0 = os:timestamp(), SW0 = os:timestamp(),
{ok, P1, {FK1, LK1}} = sst_new("../test/", "level1_src", 1, KVL1, 6000), {ok, P1, {FK1, LK1}} =
{ok, P2, {FK2, LK2}} = sst_new("../test/", "level2_src", 2, KVL2, 3000), sst_new("../test/", "level1_src", 1, KVL1, 6000, native),
{ok, P2, {FK2, LK2}} =
sst_new("../test/", "level2_src", 2, KVL2, 3000, native),
ExpFK1 = element(1, lists:nth(1, KVL1)), ExpFK1 = element(1, lists:nth(1, KVL1)),
ExpLK1 = element(1, lists:last(KVL1)), ExpLK1 = element(1, lists:last(KVL1)),
ExpFK2 = element(1, lists:nth(1, KVL2)), ExpFK2 = element(1, lists:nth(1, KVL2)),
@ -1749,7 +1831,8 @@ merge_test() ->
?assertMatch(ExpLK2, LK2), ?assertMatch(ExpLK2, LK2),
ML1 = [{next, #manifest_entry{owner = P1}, FK1}], ML1 = [{next, #manifest_entry{owner = P1}, FK1}],
ML2 = [{next, #manifest_entry{owner = P2}, FK2}], ML2 = [{next, #manifest_entry{owner = P2}, FK2}],
NewR = sst_new("../test/", "level2_merge", ML1, ML2, false, 2, N * 2), NewR =
sst_new("../test/", "level2_merge", ML1, ML2, false, 2, N * 2, native),
{ok, P3, {{Rem1, Rem2}, FK3, LK3}} = NewR, {ok, P3, {{Rem1, Rem2}, FK3, LK3}} = NewR,
?assertMatch([], Rem1), ?assertMatch([], Rem1),
?assertMatch([], Rem2), ?assertMatch([], Rem2),
@ -1783,11 +1866,8 @@ simple_persisted_range_test() ->
KVList1 = lists:ukeysort(1, KVList0), KVList1 = lists:ukeysort(1, KVList0),
[{FirstKey, _FV}|_Rest] = KVList1, [{FirstKey, _FV}|_Rest] = KVList1,
{LastKey, _LV} = lists:last(KVList1), {LastKey, _LV} = lists:last(KVList1),
{ok, Pid, {FirstKey, LastKey}} = sst_new(RP, {ok, Pid, {FirstKey, LastKey}} =
Filename, sst_new(RP, Filename, 1, KVList1, length(KVList1), native),
1,
KVList1,
length(KVList1)),
{o, B, K, null} = LastKey, {o, B, K, null} = LastKey,
SK1 = {o, B, K, 0}, SK1 = {o, B, K, 0},
@ -1836,11 +1916,8 @@ additional_range_test() ->
[], [],
lists:seq(?NOLOOK_SLOTSIZE + Gap + 1, lists:seq(?NOLOOK_SLOTSIZE + Gap + 1,
2 * ?NOLOOK_SLOTSIZE + Gap)), 2 * ?NOLOOK_SLOTSIZE + Gap)),
{ok, {ok, P1, {{Rem1, Rem2}, SK, EK}} =
P1, sst_new("../test/", "range1_src", IK1, IK2, false, 1, 9999, native),
{{Rem1, Rem2},
SK,
EK}} = sst_new("../test/", "range1_src", IK1, IK2, false, 1, 9999),
?assertMatch([], Rem1), ?assertMatch([], Rem1),
?assertMatch([], Rem2), ?assertMatch([], Rem2),
?assertMatch(SK, element(1, lists:nth(1, IK1))), ?assertMatch(SK, element(1, lists:nth(1, IK1))),
@ -1897,11 +1974,8 @@ simple_persisted_slotsize_test() ->
?LOOK_SLOTSIZE), ?LOOK_SLOTSIZE),
[{FirstKey, _FV}|_Rest] = KVList1, [{FirstKey, _FV}|_Rest] = KVList1,
{LastKey, _LV} = lists:last(KVList1), {LastKey, _LV} = lists:last(KVList1),
{ok, Pid, {FirstKey, LastKey}} = sst_new(RP, {ok, Pid, {FirstKey, LastKey}} =
Filename, sst_new(RP, Filename, 1, KVList1, length(KVList1), native),
1,
KVList1,
length(KVList1)),
lists:foreach(fun({K, V}) -> lists:foreach(fun({K, V}) ->
?assertMatch({K, V}, sst_get(Pid, K)) ?assertMatch({K, V}, sst_get(Pid, K))
end, end,
@ -1915,11 +1989,8 @@ simple_persisted_test() ->
KVList1 = lists:ukeysort(1, KVList0), KVList1 = lists:ukeysort(1, KVList0),
[{FirstKey, _FV}|_Rest] = KVList1, [{FirstKey, _FV}|_Rest] = KVList1,
{LastKey, _LV} = lists:last(KVList1), {LastKey, _LV} = lists:last(KVList1),
{ok, Pid, {FirstKey, LastKey}} = sst_new(RP, {ok, Pid, {FirstKey, LastKey}} =
Filename, sst_new(RP, Filename, 1, KVList1, length(KVList1), native),
1,
KVList1,
length(KVList1)),
SW0 = os:timestamp(), SW0 = os:timestamp(),
lists:foreach(fun({K, V}) -> lists:foreach(fun({K, V}) ->
?assertMatch({K, V}, sst_get(Pid, K)) ?assertMatch({K, V}, sst_get(Pid, K))

52
test/end_to_end/basic_SUITE.erl
View file

@ -9,7 +9,8 @@
load_and_count/1, load_and_count/1,
load_and_count_withdelete/1, load_and_count_withdelete/1,
space_clear_ondelete/1, space_clear_ondelete/1,
is_empty_test/1 is_empty_test/1,
many_put_fetch_switchcompression/1
]). ]).
all() -> [ all() -> [
@ -20,7 +21,8 @@ all() -> [
load_and_count, load_and_count,
load_and_count_withdelete, load_and_count_withdelete,
space_clear_ondelete, space_clear_ondelete,
is_empty_test is_empty_test,
many_put_fetch_switchcompression
]. ].
@ -683,3 +685,49 @@ is_empty_test(_Config) ->
true = sets:size(BLpd3()) == 0, true = sets:size(BLpd3()) == 0,
ok = leveled_bookie:book_close(Bookie1). ok = leveled_bookie:book_close(Bookie1).
many_put_fetch_switchcompression(_Config) ->
RootPath = testutil:reset_filestructure(),
StartOpts1 = [{root_path, RootPath},
{max_pencillercachesize, 16000},
{sync_strategy, riak_sync},
{compression_method, native}],
{ok, Bookie1} = leveled_bookie:book_start(StartOpts1),
{TestObject, TestSpec} = testutil:generate_testobject(),
ok = testutil:book_riakput(Bookie1, TestObject, TestSpec),
testutil:check_forobject(Bookie1, TestObject),
ok = leveled_bookie:book_close(Bookie1),
StartOpts2 = [{root_path, RootPath},
{max_journalsize, 500000000},
{max_pencillercachesize, 32000},
{sync_strategy, testutil:sync_strategy()},
{compression_method, lz4}],
%% Change compression method
{ok, Bookie2} = leveled_bookie:book_start(StartOpts2),
testutil:check_forobject(Bookie2, TestObject),
GenList = [2, 40002, 80002, 120002],
CLs = testutil:load_objects(40000, GenList, Bookie2, TestObject,
fun testutil:generate_smallobjects/2),
CL1A = lists:nth(1, CLs),
ChkListFixed = lists:nth(length(CLs), CLs),
testutil:check_forlist(Bookie2, CL1A),
ObjList2A = testutil:generate_objects(5000, 2),
testutil:riakload(Bookie2, ObjList2A),
ChkList2A = lists:sublist(lists:sort(ObjList2A), 1000),
testutil:check_forlist(Bookie2, ChkList2A),
testutil:check_forlist(Bookie2, ChkListFixed),
testutil:check_forobject(Bookie2, TestObject),
testutil:check_forlist(Bookie2, ChkList2A),
testutil:check_forlist(Bookie2, ChkListFixed),
testutil:check_forobject(Bookie2, TestObject),
ok = leveled_bookie:book_close(Bookie2),
%% Change method back again
{ok, Bookie3} = leveled_bookie:book_start(StartOpts1),
testutil:check_forlist(Bookie3, ChkList2A),
testutil:check_forlist(Bookie3, ChkListFixed),
testutil:check_forobject(Bookie3, TestObject),
testutil:check_formissingobject(Bookie3, "Bookie1", "MissingKey0123"),
ok = leveled_bookie:book_destroy(Bookie3).