umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Rename to pmanifest

Browse source 
There may be a seperate imanifest in the future
This commit is contained in:
martinsumner 2017-01-17 14:11:50 +00:00
parent 2e734ff501
commit 7086717765
3 changed files with 37 additions and 37 deletions
Download patch file Download diff file Expand all files Collapse all files

34
src/leveled_pclerk.erl
View file

@ -125,15 +125,15 @@ handle_work({SrcLevel, Manifest}, State) ->
UpdManifest), UpdManifest),
leveled_log:log_timer("PC017", [], SWMC), leveled_log:log_timer("PC017", [], SWMC),
SWSM = os:timestamp(), SWSM = os:timestamp(),
ok = leveled_manifest:save_manifest(UpdManifest, ok = leveled_pmanifest:save_manifest(UpdManifest,
State#state.root_path), State#state.root_path),
leveled_log:log_timer("PC018", [], SWSM), leveled_log:log_timer("PC018", [], SWSM),
{leveled_manifest:get_manifest_sqn(UpdManifest), EntriesToDelete}. {leveled_pmanifest:get_manifest_sqn(UpdManifest), EntriesToDelete}.
merge(SrcLevel, Manifest, RootPath) -> merge(SrcLevel, Manifest, RootPath) ->
Src = leveled_manifest:mergefile_selector(Manifest, SrcLevel), Src = leveled_pmanifest:mergefile_selector(Manifest, SrcLevel),
NewSQN = leveled_manifest:get_manifest_sqn(Manifest) + 1, NewSQN = leveled_pmanifest:get_manifest_sqn(Manifest) + 1,
SinkList = leveled_manifest:merge_lookup(Manifest, SinkList = leveled_pmanifest:merge_lookup(Manifest,
SrcLevel + 1, SrcLevel + 1,
Src#manifest_entry.start_key, Src#manifest_entry.start_key,
Src#manifest_entry.end_key), Src#manifest_entry.end_key),
@ -143,7 +143,7 @@ merge(SrcLevel, Manifest, RootPath) ->
0 -> 0 ->
leveled_log:log("PC009", leveled_log:log("PC009",
[Src#manifest_entry.filename, SrcLevel + 1]), [Src#manifest_entry.filename, SrcLevel + 1]),
Man0 = leveled_manifest:switch_manifest_entry(Manifest, Man0 = leveled_pmanifest:switch_manifest_entry(Manifest,
NewSQN, NewSQN,
SrcLevel, SrcLevel,
Src), Src),
@ -172,7 +172,7 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) ->
SrcList = [{next, Src, all}], SrcList = [{next, Src, all}],
MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner), MaxSQN = leveled_sst:sst_getmaxsequencenumber(Src#manifest_entry.owner),
SinkLevel = SrcLevel + 1, SinkLevel = SrcLevel + 1,
SinkBasement = leveled_manifest:is_basement(Manifest, SinkLevel), SinkBasement = leveled_pmanifest:is_basement(Manifest, SinkLevel),
Additions = do_merge(SrcList, SinkList, Additions = do_merge(SrcList, SinkList,
SinkLevel, SinkBasement, SinkLevel, SinkBasement,
RootPath, NewSQN, MaxSQN, RootPath, NewSQN, MaxSQN,
@ -182,16 +182,16 @@ perform_merge(Manifest, Src, SinkList, SrcLevel, RootPath, NewSQN) ->
ME ME
end, end,
SinkManifestList = lists:map(RevertPointerFun, SinkList), SinkManifestList = lists:map(RevertPointerFun, SinkList),
Man0 = leveled_manifest:remove_manifest_entry(Manifest, Man0 = leveled_pmanifest:remove_manifest_entry(Manifest,
NewSQN, NewSQN,
SinkLevel, SinkLevel,
SinkManifestList), SinkManifestList),
Man1 = leveled_manifest:insert_manifest_entry(Man0, Man1 = leveled_pmanifest:insert_manifest_entry(Man0,
NewSQN, NewSQN,
SinkLevel, SinkLevel,
Additions), Additions),
Man2 = leveled_manifest:remove_manifest_entry(Man1, Man2 = leveled_pmanifest:remove_manifest_entry(Man1,
NewSQN, NewSQN,
SrcLevel, SrcLevel,
Src), Src),
@ -297,18 +297,18 @@ merge_file_test() ->
end_key = lists:last(KL4_L2), end_key = lists:last(KL4_L2),
start_key = lists:nth(1, KL4_L2)}, start_key = lists:nth(1, KL4_L2)},
Man0 = leveled_manifest:new_manifest(), Man0 = leveled_pmanifest:new_manifest(),
Man1 = leveled_manifest:insert_manifest_entry(Man0, 1, 2, E2), Man1 = leveled_pmanifest:insert_manifest_entry(Man0, 1, 2, E2),
Man2 = leveled_manifest:insert_manifest_entry(Man1, 1, 2, E3), Man2 = leveled_pmanifest:insert_manifest_entry(Man1, 1, 2, E3),
Man3 = leveled_manifest:insert_manifest_entry(Man2, 1, 2, E4), Man3 = leveled_pmanifest:insert_manifest_entry(Man2, 1, 2, E4),
Man4 = leveled_manifest:insert_manifest_entry(Man3, 1, 2, E5), Man4 = leveled_pmanifest:insert_manifest_entry(Man3, 1, 2, E5),
Man5 = leveled_manifest:insert_manifest_entry(Man4, 2, 1, E1), Man5 = leveled_pmanifest:insert_manifest_entry(Man4, 2, 1, E1),
PointerList = lists:map(fun(ME) -> {next, ME, all} end, PointerList = lists:map(fun(ME) -> {next, ME, all} end,
[E2, E3, E4, E5]), [E2, E3, E4, E5]),
{Man6, _Dels} = perform_merge(Man5, E1, PointerList, 1, "../test", 3), {Man6, _Dels} = perform_merge(Man5, E1, PointerList, 1, "../test", 3),
?assertMatch(3, leveled_manifest:get_manifest_sqn(Man6)). ?assertMatch(3, leveled_pmanifest:get_manifest_sqn(Man6)).
coverage_cheat_test() -> coverage_cheat_test() ->
{ok, _State1} = code_change(null, #state{}, null). {ok, _State1} = code_change(null, #state{}, null).

38
src/leveled_penciller.erl
View file

@ -329,7 +329,7 @@ init([PCLopts]) ->
{undefined, true} -> {undefined, true} ->
SrcPenciller = PCLopts#penciller_options.source_penciller, SrcPenciller = PCLopts#penciller_options.source_penciller,
{ok, State} = pcl_registersnapshot(SrcPenciller, self()), {ok, State} = pcl_registersnapshot(SrcPenciller, self()),
ManifestClone = leveled_manifest:copy_manifest(State#state.manifest), ManifestClone = leveled_pmanifest:copy_manifest(State#state.manifest),
leveled_log:log("P0001", [self()]), leveled_log:log("P0001", [self()]),
{ok, State#state{is_snapshot=true, {ok, State#state{is_snapshot=true,
source_penciller=SrcPenciller, source_penciller=SrcPenciller,
@ -411,7 +411,7 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc, MaxKeys},
SetupFoldFun = SetupFoldFun =
fun(Level, Acc) -> fun(Level, Acc) ->
Pointers = leveled_manifest:range_lookup(State#state.manifest, Pointers = leveled_pmanifest:range_lookup(State#state.manifest,
Level, Level,
StartKey, StartKey,
EndKey), EndKey),
@ -431,7 +431,7 @@ handle_call({fetch_keys, StartKey, EndKey, AccFun, InitAcc, MaxKeys},
handle_call(get_startup_sqn, _From, State) -> handle_call(get_startup_sqn, _From, State) ->
{reply, State#state.persisted_sqn, State}; {reply, State#state.persisted_sqn, State};
handle_call({register_snapshot, Snapshot}, _From, State) -> handle_call({register_snapshot, Snapshot}, _From, State) ->
Manifest0 = leveled_manifest:add_snapshot(State#state.manifest, Manifest0 = leveled_pmanifest:add_snapshot(State#state.manifest,
Snapshot, Snapshot,
?SNAPSHOT_TIMEOUT), ?SNAPSHOT_TIMEOUT),
{reply, {ok, State}, State#state{manifest = Manifest0}}; {reply, {ok, State}, State#state{manifest = Manifest0}};
@ -461,11 +461,11 @@ handle_call(doom, _From, State) ->
{stop, normal, {ok, [ManifestFP, FilesFP]}, State}. {stop, normal, {ok, [ManifestFP, FilesFP]}, State}.
handle_cast({manifest_change, NewManifest}, State) -> handle_cast({manifest_change, NewManifest}, State) ->
NewManSQN = leveled_manifest:get_manifest_sqn(NewManifest), NewManSQN = leveled_pmanifest:get_manifest_sqn(NewManifest),
ok = leveled_pclerk:clerk_promptdeletions(State#state.clerk, NewManSQN), ok = leveled_pclerk:clerk_promptdeletions(State#state.clerk, NewManSQN),
{noreply, State#state{manifest = NewManifest, work_ongoing=false}}; {noreply, State#state{manifest = NewManifest, work_ongoing=false}};
handle_cast({release_snapshot, Snapshot}, State) -> handle_cast({release_snapshot, Snapshot}, State) ->
Manifest0 = leveled_manifest:release_snapshot(State#state.manifest, Manifest0 = leveled_pmanifest:release_snapshot(State#state.manifest,
Snapshot), Snapshot),
leveled_log:log("P0003", [Snapshot]), leveled_log:log("P0003", [Snapshot]),
{noreply, State#state{manifest=Manifest0}}; {noreply, State#state{manifest=Manifest0}};
@ -473,7 +473,7 @@ handle_cast({confirm_delete, Filename, FilePid}, State=#state{is_snapshot=Snap})
when Snap == false -> when Snap == false ->
case State#state.work_ongoing of case State#state.work_ongoing of
false -> false ->
R2D = leveled_manifest:ready_to_delete(State#state.manifest, R2D = leveled_pmanifest:ready_to_delete(State#state.manifest,
Filename), Filename),
case R2D of case R2D of
{true, M0} -> {true, M0} ->
@ -495,8 +495,8 @@ handle_cast({levelzero_complete, FN, StartKey, EndKey}, State) ->
end_key=EndKey, end_key=EndKey,
owner=State#state.levelzero_constructor, owner=State#state.levelzero_constructor,
filename=FN}, filename=FN},
ManifestSQN = leveled_manifest:get_manifest_sqn(State#state.manifest) + 1, ManifestSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest) + 1,
UpdMan = leveled_manifest:insert_manifest_entry(State#state.manifest, UpdMan = leveled_pmanifest:insert_manifest_entry(State#state.manifest,
ManifestSQN, ManifestSQN,
0, 0,
ManEntry), ManEntry),
@ -515,7 +515,7 @@ handle_cast(work_for_clerk, State) ->
true -> true ->
{noreply, State}; {noreply, State};
false -> false ->
{WL, WC} = leveled_manifest:check_for_work(State#state.manifest, {WL, WC} = leveled_pmanifest:check_for_work(State#state.manifest,
?LEVEL_SCALEFACTOR), ?LEVEL_SCALEFACTOR),
case WC of case WC of
0 -> 0 ->
@ -558,7 +558,7 @@ terminate(Reason, State) ->
ok = leveled_pclerk:clerk_close(State#state.clerk), ok = leveled_pclerk:clerk_close(State#state.clerk),
leveled_log:log("P0008", [Reason]), leveled_log:log("P0008", [Reason]),
L0_Present = leveled_manifest:key_lookup(State#state.manifest, 0, all), L0_Present = leveled_pmanifest:key_lookup(State#state.manifest, 0, all),
L0_Left = State#state.levelzero_size > 0, L0_Left = State#state.levelzero_size > 0,
case {State#state.levelzero_pending, L0_Present, L0_Left} of case {State#state.levelzero_pending, L0_Present, L0_Left} of
{false, false, true} -> {false, false, true} ->
@ -573,7 +573,7 @@ terminate(Reason, State) ->
fun(ME) -> fun(ME) ->
ok = leveled_sst:sst_close(ME#manifest_entry.owner) ok = leveled_sst:sst_close(ME#manifest_entry.owner)
end, end,
leveled_manifest:close_manifest(State#state.manifest, EntryCloseFun), leveled_pmanifest:close_manifest(State#state.manifest, EntryCloseFun),
leveled_log:log("P0011", []), leveled_log:log("P0011", []),
ok. ok.
@ -610,18 +610,18 @@ start_from_file(PCLopts) ->
levelzero_index=leveled_pmem:new_index()}, levelzero_index=leveled_pmem:new_index()},
%% Open manifest %% Open manifest
Manifest0 = leveled_manifest:open_manifest(RootPath), Manifest0 = leveled_pmanifest:open_manifest(RootPath),
OpenFun = OpenFun =
fun(FN) -> fun(FN) ->
{ok, Pid, {_FK, _LK}} = leveled_sst:sst_open(FN), {ok, Pid, {_FK, _LK}} = leveled_sst:sst_open(FN),
Pid Pid
end, end,
SQNFun = fun leveled_sst:sst_getmaxsequencenumber/1, SQNFun = fun leveled_sst:sst_getmaxsequencenumber/1,
{MaxSQN, Manifest1} = leveled_manifest:load_manifest(Manifest0, {MaxSQN, Manifest1} = leveled_pmanifest:load_manifest(Manifest0,
OpenFun, OpenFun,
SQNFun), SQNFun),
leveled_log:log("P0014", [MaxSQN]), leveled_log:log("P0014", [MaxSQN]),
ManSQN = leveled_manifest:get_manifest_sqn(Manifest1), ManSQN = leveled_pmanifest:get_manifest_sqn(Manifest1),
leveled_log:log("P0035", [ManSQN]), leveled_log:log("P0035", [ManSQN]),
%% Find any L0 files %% Find any L0 files
L0FN = filepath(RootPath, ManSQN + 1, new_merge_files) ++ "_0_0.sst", L0FN = filepath(RootPath, ManSQN + 1, new_merge_files) ++ "_0_0.sst",
@ -636,7 +636,7 @@ start_from_file(PCLopts) ->
end_key = L0EndKey, end_key = L0EndKey,
filename = L0FN, filename = L0FN,
owner = L0Pid}, owner = L0Pid},
Manifest2 = leveled_manifest:insert_manifest_entry(Manifest1, Manifest2 = leveled_pmanifest:insert_manifest_entry(Manifest1,
ManSQN + 1, ManSQN + 1,
0, 0,
L0Entry), L0Entry),
@ -675,7 +675,7 @@ update_levelzero(L0Size, {PushedTree, PushedIdx, MinSQN, MaxSQN},
ledger_sqn=UpdMaxSQN}, ledger_sqn=UpdMaxSQN},
CacheTooBig = NewL0Size > State#state.levelzero_maxcachesize, CacheTooBig = NewL0Size > State#state.levelzero_maxcachesize,
CacheMuchTooBig = NewL0Size > ?SUPER_MAX_TABLE_SIZE, CacheMuchTooBig = NewL0Size > ?SUPER_MAX_TABLE_SIZE,
L0Free = not leveled_manifest:levelzero_present(State#state.manifest), L0Free = not leveled_pmanifest:levelzero_present(State#state.manifest),
RandomFactor = RandomFactor =
case State#state.levelzero_cointoss of case State#state.levelzero_cointoss of
true -> true ->
@ -735,7 +735,7 @@ roll_memory(State, true) ->
Constructor. Constructor.
levelzero_filename(State) -> levelzero_filename(State) ->
ManSQN = leveled_manifest:get_manifest_sqn(State#state.manifest) + 1, ManSQN = leveled_pmanifest:get_manifest_sqn(State#state.manifest) + 1,
FileName = State#state.root_path FileName = State#state.root_path
++ "/" ++ ?FILES_FP ++ "/" ++ "/" ++ ?FILES_FP ++ "/"
++ integer_to_list(ManSQN) ++ "_0_0", ++ integer_to_list(ManSQN) ++ "_0_0",
@ -770,7 +770,7 @@ fetch_mem(Key, Hash, Manifest, L0Cache, L0Index) ->
fetch(_Key, _Hash, _Manifest, ?MAX_LEVELS + 1, _FetchFun) -> fetch(_Key, _Hash, _Manifest, ?MAX_LEVELS + 1, _FetchFun) ->
{not_present, basement}; {not_present, basement};
fetch(Key, Hash, Manifest, Level, FetchFun) -> fetch(Key, Hash, Manifest, Level, FetchFun) ->
case leveled_manifest:key_lookup(Manifest, Level, Key) of case leveled_pmanifest:key_lookup(Manifest, Level, Key) of
false -> false ->
fetch(Key, Hash, Manifest, Level + 1, FetchFun); fetch(Key, Hash, Manifest, Level + 1, FetchFun);
FP -> FP ->
@ -1053,7 +1053,7 @@ generate_randomkeys(Count, SQN, Acc) ->
clean_testdir(RootPath) -> clean_testdir(RootPath) ->
clean_subdir(leveled_manifest:filepath(RootPath, manifest)), clean_subdir(leveled_pmanifest:filepath(RootPath, manifest)),
clean_subdir(filepath(RootPath, files)). clean_subdir(filepath(RootPath, files)).
clean_subdir(DirPath) -> clean_subdir(DirPath) ->

2
src/leveled_manifest.erl → src/leveled_pmanifest.erl
View file

@ -5,7 +5,7 @@
%% %%
-module(leveled_manifest). -module(leveled_pmanifest).
-include("include/leveled.hrl"). -include("include/leveled.hrl").