umgeher/leveled
0
Fork 0
Code Issues Pull requests Releases Activity Actions

Failed attempt to hack in LZ4

Browse source
This commit is contained in:
Martin Sumner 2017-11-03 11:47:00 +00:00
parent c6749e61a9
commit 912920a53c
18 changed files with 7603 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

15
src/leveled_cdb.erl
View file

@ -1385,14 +1385,13 @@ perform_write_hash_tables(Handle, HashTreeBin, StartPos) ->
%% The List passed in should be made up of {Index, Position, Count} tuples
write_top_index_table(Handle, BasePos, IndexList) ->
FnWriteIndex = fun({_Index, Pos, Count}, {AccBin, CurrPos}) ->
case Count == 0 of
true ->
PosLE = endian_flip(CurrPos),
NextPos = CurrPos;
false ->
PosLE = endian_flip(Pos),
NextPos = Pos + (Count * ?DWORD_SIZE)
end,
{PosLE, NextPos} =
case Count == 0 of
true ->
{endian_flip(CurrPos), CurrPos};
false ->
{endian_flip(Pos), Pos + (Count * ?DWORD_SIZE)}
end,
CountLE = endian_flip(Count),
{<<AccBin/binary, PosLE:32, CountLE:32>>, NextPos}
end,

64
src/lz4_nif.erl Normal file
View file

@ -0,0 +1,64 @@
%% Copyright (c) 2017-Present Pivotal Software, Inc. All rights reserved.
%%
%% This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
%% Public License 1.1 ("MPL") and the Apache License version 2
%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
%% please see LICENSE-APACHE2.
%%
%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
%% either express or implied. See the LICENSE file for specific language governing
%% rights and limitations of this software.
%%
%% If you have any questions regarding licensing, please contact us at
%% info@rabbitmq.com.
-module(lz4_nif).
%% lz4f.
-export([lz4f_compress_frame/2]).
-export([lz4f_create_compression_context/0]).
-export([lz4f_compress_begin/2]).
-export([lz4f_compress_update/2]).
-export([lz4f_flush/1]).
-export([lz4f_compress_end/1]).
-export([lz4f_create_decompression_context/0]).
-export([lz4f_get_frame_info/2]).
-export([lz4f_decompress/2]).
-on_load(on_load/0).
on_load() ->
case code:priv_dir(leveled) of
{error, _} ->
{error, {load_failed, "Could not determine the leveled priv/ directory."}};
Path ->
erlang:load_nif(filename:join(Path, atom_to_list(?MODULE)), 0)
end.
%% lz4f.
lz4f_compress_frame(_, _) ->
erlang:nif_error({not_loaded, ?MODULE}).
lz4f_create_compression_context() ->
erlang:nif_error({not_loaded, ?MODULE}).
lz4f_compress_begin(_, _) ->
erlang:nif_error({not_loaded, ?MODULE}).
lz4f_compress_update(_, _) ->
erlang:nif_error({not_loaded, ?MODULE}).
lz4f_flush(_) ->
erlang:nif_error({not_loaded, ?MODULE}).
lz4f_compress_end(_) ->
erlang:nif_error({not_loaded, ?MODULE}).
lz4f_create_decompression_context() ->
erlang:nif_error({not_loaded, ?MODULE}).
lz4f_get_frame_info(_, _) ->
erlang:nif_error({not_loaded, ?MODULE}).
lz4f_decompress(_, _) ->
erlang:nif_error({not_loaded, ?MODULE}).

107
src/lz4f.erl Normal file
View file

@ -0,0 +1,107 @@
%% Copyright (c) 2017-Present Pivotal Software, Inc. All rights reserved.
%%
%% This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
%% Public License 1.1 ("MPL") and the Apache License version 2
%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
%% please see LICENSE-APACHE2.
%%
%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
%% either express or implied. See the LICENSE file for specific language governing
%% rights and limitations of this software.
%%
%% If you have any questions regarding licensing, please contact us at
%% info@rabbitmq.com.
-module(lz4f).
%% Simple compression.
-export([compress_frame/1]).
-export([compress_frame/2]).
%% Advanced compression.
-export([create_compression_context/0]).
-export([compress_begin/1]).
-export([compress_begin/2]).
-export([compress_update/2]).
-export([flush/1]).
-export([compress_end/1]).
%% Decompression.
-export([create_decompression_context/0]).
-export([get_frame_info/2]).
-export([decompress/1]).
-export([decompress/2]).
-type block_size_id() :: default | max64KB | max256KB | max1MB | max4MB.
-type block_mode() :: linked | independent.
-type frame_type() :: frame | skippable_frame.
-type frame_info() :: #{
block_size_id => block_size_id(),
block_mode => block_mode(),
content_checksum => boolean(),
frame_type => frame_type(),
content_size => non_neg_integer()
}.
-type opts() :: #{
frame_info => frame_info(),
compression_level => 0..16,
auto_flush => boolean()
}.
-opaque cctx() :: <<>>. %% Resource.
-export_type([cctx/0]).
-opaque dctx() :: <<>>. %% Resource.
-export_type([dctx/0]).
-spec compress_frame(binary()) -> binary().
compress_frame(Data) ->
lz4_nif:lz4f_compress_frame(Data, #{}).
-spec compress_frame(binary(), opts()) -> binary().
compress_frame(Data, Opts) ->
lz4_nif:lz4f_compress_frame(Data, Opts).
-spec create_compression_context() -> cctx().
create_compression_context() ->
lz4_nif:lz4f_create_compression_context().
-spec compress_begin(cctx()) -> binary().
compress_begin(Cctx) ->
compress_begin(Cctx, #{}).
-spec compress_begin(cctx(), opts()) -> binary().
compress_begin(Cctx, Opts) ->
lz4_nif:lz4f_compress_begin(Cctx, Opts).
-spec compress_update(cctx(), binary()) -> binary().
compress_update(Cctx, Data) ->
lz4_nif:lz4f_compress_update(Cctx, Data).
-spec flush(cctx()) -> binary().
flush(Cctx) ->
lz4_nif:lz4f_flush(Cctx).
-spec compress_end(cctx()) -> binary().
compress_end(Cctx) ->
lz4_nif:lz4f_compress_end(Cctx).
-spec create_decompression_context() -> dctx().
create_decompression_context() ->
lz4_nif:lz4f_create_decompression_context().
-spec get_frame_info(dctx(), binary()) -> {ok, frame_info(), non_neg_integer()}.
get_frame_info(Dctx, Data) ->
lz4_nif:lz4f_get_frame_info(Dctx, Data).
-spec decompress(binary()) -> iolist().
decompress(Data) ->
decompress(create_decompression_context(), Data).
-spec decompress(dctx(), binary()) -> iolist().
decompress(Dctx, Data) ->
lz4_nif:lz4f_decompress(Dctx, Data).
%% @todo LZ4F_resetDecompressionContext