umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Reduce compaction targets

Browse source 
Cmpaction is overly aggressive.  It is a lot of work to compact a run of
files for just 20% reduction in disk space, when disk space for the
Journal (i.e. low IOPS disk space should be relatively inexpensive).
Require at least a 40% reduction for a compaction job.
This commit is contained in:
martinsumner 2017-03-30 12:15:36 +01:00
parent 512af7f41a
commit 11ff3129f3
2 changed files with 42 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

82
src/leveled_iclerk.erl
View file

@ -95,8 +95,8 @@
%% How many consecutive files to compact in one run %% How many consecutive files to compact in one run
-define(MAX_COMPACTION_RUN, 4). -define(MAX_COMPACTION_RUN, 4).
%% Sliding scale to allow preference of longer runs up to maximum %% Sliding scale to allow preference of longer runs up to maximum
-define(SINGLEFILE_COMPACTION_TARGET, 60.0). -define(SINGLEFILE_COMPACTION_TARGET, 40.0).
-define(MAXRUN_COMPACTION_TARGET, 80.0). -define(MAXRUN_COMPACTION_TARGET, 60.0).
-define(CRC_SIZE, 4). -define(CRC_SIZE, 4).
-define(DEFAULT_RELOAD_STRATEGY, leveled_codec:inker_reload_strategy([])). -define(DEFAULT_RELOAD_STRATEGY, leveled_codec:inker_reload_strategy([])).
-define(DEFAULT_WASTE_RETENTION_PERIOD, 86400). -define(DEFAULT_WASTE_RETENTION_PERIOD, 86400).
@ -550,20 +550,20 @@ simple_score_test() ->
#candidate{compaction_perc = 75.0}, #candidate{compaction_perc = 75.0},
#candidate{compaction_perc = 76.0}, #candidate{compaction_perc = 76.0},
#candidate{compaction_perc = 70.0}], #candidate{compaction_perc = 70.0}],
?assertMatch(6.0, score_run(Run1, 4)), ?assertMatch(-14.0, score_run(Run1, 4)),
Run2 = [#candidate{compaction_perc = 75.0}], Run2 = [#candidate{compaction_perc = 75.0}],
?assertMatch(-15.0, score_run(Run2, 4)), ?assertMatch(-35.0, score_run(Run2, 4)),
?assertMatch(0.0, score_run([], 4)), ?assertMatch(0.0, score_run([], 4)),
Run3 = [#candidate{compaction_perc = 100.0}], Run3 = [#candidate{compaction_perc = 100.0}],
?assertMatch(-40.0, score_run(Run3, 4)). ?assertMatch(-60.0, score_run(Run3, 4)).
score_compare_test() -> score_compare_test() ->
Run1 = [#candidate{compaction_perc = 75.0}, Run1 = [#candidate{compaction_perc = 55.0},
#candidate{compaction_perc = 75.0}, #candidate{compaction_perc = 55.0},
#candidate{compaction_perc = 76.0}, #candidate{compaction_perc = 56.0},
#candidate{compaction_perc = 70.0}], #candidate{compaction_perc = 50.0}],
?assertMatch(6.0, score_run(Run1, 4)), ?assertMatch(6.0, score_run(Run1, 4)),
Run2 = [#candidate{compaction_perc = 75.0}], Run2 = [#candidate{compaction_perc = 55.0}],
?assertMatch(Run1, choose_best_assessment(Run1, Run2, 4)), ?assertMatch(Run1, choose_best_assessment(Run1, Run2, 4)),
?assertMatch(Run2, choose_best_assessment(Run1 ++ Run2, Run2, 4)). ?assertMatch(Run2, choose_best_assessment(Run1 ++ Run2, Run2, 4)).
@ -585,27 +585,27 @@ file_gc_test() ->
find_bestrun_test() -> find_bestrun_test() ->
%% Tests dependent on these defaults %% Tests dependent on these defaults
%% -define(MAX_COMPACTION_RUN, 4). %% -define(MAX_COMPACTION_RUN, 4).
%% -define(SINGLEFILE_COMPACTION_TARGET, 60.0). %% -define(SINGLEFILE_COMPACTION_TARGET, 40.0).
%% -define(MAXRUN_COMPACTION_TARGET, 80.0). %% -define(MAXRUN_COMPACTION_TARGET, 60.0).
%% Tested first with blocks significant as no back-tracking %% Tested first with blocks significant as no back-tracking
Block1 = [#candidate{compaction_perc = 75.0}, Block1 = [#candidate{compaction_perc = 55.0},
#candidate{compaction_perc = 85.0}, #candidate{compaction_perc = 65.0},
#candidate{compaction_perc = 62.0}, #candidate{compaction_perc = 42.0},
#candidate{compaction_perc = 70.0}], #candidate{compaction_perc = 50.0}],
Block2 = [#candidate{compaction_perc = 58.0}, Block2 = [#candidate{compaction_perc = 38.0},
#candidate{compaction_perc = 95.0}, #candidate{compaction_perc = 75.0},
#candidate{compaction_perc = 95.0}, #candidate{compaction_perc = 75.0},
#candidate{compaction_perc = 65.0}], #candidate{compaction_perc = 45.0}],
Block3 = [#candidate{compaction_perc = 90.0}, Block3 = [#candidate{compaction_perc = 70.0},
#candidate{compaction_perc = 100.0}, #candidate{compaction_perc = 100.0},
#candidate{compaction_perc = 100.0}, #candidate{compaction_perc = 100.0},
#candidate{compaction_perc = 100.0}], #candidate{compaction_perc = 100.0}],
Block4 = [#candidate{compaction_perc = 75.0}, Block4 = [#candidate{compaction_perc = 55.0},
#candidate{compaction_perc = 76.0}, #candidate{compaction_perc = 56.0},
#candidate{compaction_perc = 76.0}, #candidate{compaction_perc = 56.0},
#candidate{compaction_perc = 40.0}],
Block5 = [#candidate{compaction_perc = 60.0},
#candidate{compaction_perc = 60.0}], #candidate{compaction_perc = 60.0}],
Block5 = [#candidate{compaction_perc = 80.0},
#candidate{compaction_perc = 80.0}],
CList0 = Block1 ++ Block2 ++ Block3 ++ Block4 ++ Block5, CList0 = Block1 ++ Block2 ++ Block3 ++ Block4 ++ Block5,
?assertMatch(Block4, assess_candidates(CList0, 4, [], [])), ?assertMatch(Block4, assess_candidates(CList0, 4, [], [])),
CList1 = CList0 ++ [#candidate{compaction_perc = 20.0}], CList1 = CList0 ++ [#candidate{compaction_perc = 20.0}],
@ -614,26 +614,26 @@ find_bestrun_test() ->
CList2 = Block4 ++ Block3 ++ Block2 ++ Block1 ++ Block5, CList2 = Block4 ++ Block3 ++ Block2 ++ Block1 ++ Block5,
?assertMatch(Block4, assess_candidates(CList2, 4, [], [])), ?assertMatch(Block4, assess_candidates(CList2, 4, [], [])),
CList3 = Block5 ++ Block1 ++ Block2 ++ Block3 ++ Block4, CList3 = Block5 ++ Block1 ++ Block2 ++ Block3 ++ Block4,
?assertMatch([#candidate{compaction_perc = 62.0}, ?assertMatch([#candidate{compaction_perc = 42.0},
#candidate{compaction_perc = 70.0}, #candidate{compaction_perc = 50.0},
#candidate{compaction_perc = 58.0}], #candidate{compaction_perc = 38.0}],
assess_candidates(CList3, 4, [], [])), assess_candidates(CList3, 4, [], [])),
%% Now do some back-tracking to get a genuinely optimal solution without %% Now do some back-tracking to get a genuinely optimal solution without
%% needing to re-order %% needing to re-order
?assertMatch([#candidate{compaction_perc = 62.0}, ?assertMatch([#candidate{compaction_perc = 42.0},
#candidate{compaction_perc = 70.0}, #candidate{compaction_perc = 50.0},
#candidate{compaction_perc = 58.0}], #candidate{compaction_perc = 38.0}],
assess_candidates(CList0, 4)), assess_candidates(CList0, 4)),
?assertMatch([#candidate{compaction_perc = 62.0}, ?assertMatch([#candidate{compaction_perc = 42.0},
#candidate{compaction_perc = 70.0}, #candidate{compaction_perc = 50.0},
#candidate{compaction_perc = 58.0}], #candidate{compaction_perc = 38.0}],
assess_candidates(CList0, 5)), assess_candidates(CList0, 5)),
?assertMatch([#candidate{compaction_perc = 62.0}, ?assertMatch([#candidate{compaction_perc = 42.0},
#candidate{compaction_perc = 70.0}, #candidate{compaction_perc = 50.0},
#candidate{compaction_perc = 58.0}, #candidate{compaction_perc = 38.0},
#candidate{compaction_perc = 95.0}, #candidate{compaction_perc = 75.0},
#candidate{compaction_perc = 95.0}, #candidate{compaction_perc = 75.0},
#candidate{compaction_perc = 65.0}], #candidate{compaction_perc = 45.0}],
assess_candidates(CList0, 6)). assess_candidates(CList0, 6)).
test_ledgerkey(Key) -> test_ledgerkey(Key) ->

2
src/leveled_inker.erl
View file

@ -897,7 +897,7 @@ compact_journal_test() ->
end end, end end,
false, false,
CompactedManifest2), CompactedManifest2),
?assertMatch(true, R), ?assertMatch(false, R),
?assertMatch(2, length(CompactedManifest2)), ?assertMatch(2, length(CompactedManifest2)),
ink_close(Ink1), ink_close(Ink1),
clean_testdir(RootPath). clean_testdir(RootPath).