Failed attempt to hack in LZ4

.erlang.mk/deps.log

@@ -0,0 +1,2 @@
+/Users/martinsumner/dbroot/leveled/deps/lz4_src
+/Users/martinsumner/dbroot/leveled/deps/nif_helpers

Makefile

@@ -0,0 +1,41 @@
+PROJECT = lz4
+PROJECT_DESCRIPTION = New project
+PROJECT_VERSION = 0.1.0
+
+BUILD_DEPS = lz4_src nif_helpers
+dep_lz4_src = git https://github.com/lz4/lz4 v1.8.0
+dep_nif_helpers = git https://github.com/ninenines/nif_helpers master
+DEP_PLUGINS = nif_helpers
+
+C_SRC_OUTPUT = $(CURDIR)/priv/lz4_nif
+
+TEST_DEPS = ct_helper
+dep_ct_helper = git https://github.com/extend/ct_helper master
+
+include erlang.mk
+
+CFLAGS += -I $(DEPS_DIR)/lz4_src/lib
+# This is required in order to build a liblz4.a that we can
+# include in our shared library.
+export CPPFLAGS += -shared -fPIC
+LDLIBS += $(DEPS_DIR)/lz4_src/lib/liblz4.a
+
+-include c_src/env.mk
+
+cppcheck:
+	cppcheck -f --quiet --error-exitcode=2 --enable=all --inconclusive --std=posix \
+        -Ideps/lz4_src/lib/ -Ideps/nif_helpers/ -I$(ERTS_INCLUDE_DIR) c_src/
+
+scan-build:
+	make clean
+	scan-build make
+
+# Download a large file for use in compression tests.
+
+PDF_REFERENCE = http://www.adobe.com/content/dam/Adobe/en/devnet/acrobat/pdfs/pdf_reference_1-7.pdf
+
+test-build:: $(TEST_DIR)/lz4f_SUITE_data/pdf_reference_1-7.pdf
+
+$(TEST_DIR)/lz4f_SUITE_data/pdf_reference_1-7.pdf:
+	$(verbose) mkdir -p $(TEST_DIR)/lz4f_SUITE_data/
+	$(gen_verbose) $(call core_http_get,$@,$(PDF_REFERENCE))

c_src/env.mk

@@ -0,0 +1,3 @@
+ERTS_INCLUDE_DIR ?= /Users/martinsumner/erlang/18.3/erts-7.3/include/
+ERL_INTERFACE_INCLUDE_DIR ?= /Users/martinsumner/erlang/18.3/lib/erl_interface-3.8.2/include
+ERL_INTERFACE_LIB_DIR ?= /Users/martinsumner/erlang/18.3/lib/erl_interface-3.8.2/lib

c_src/lz4.c

@@ -0,0 +1,47 @@
+// Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+//
+// This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
+// Public License 1.1 ("MPL") and the Apache License version 2
+// ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+// please see LICENSE-APACHE2.
+//
+// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+// either express or implied. See the LICENSE file for specific language governing
+// rights and limitations of this software.
+//
+// If you have any questions regarding licensing, please contact us at
+// info@rabbitmq.com.
+
+#include "lz4_erlang.h"
+
+NIF_ATOMS(NIF_ATOM_DECL)
+NIF_RESOURCES(NIF_RES_DECL)
+
+int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info);
+int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
+void unload(ErlNifEnv* env, void* priv_data);
+
+int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    NIF_ATOMS(NIF_ATOM_INIT)
+    NIF_RESOURCES(NIF_RES_INIT)
+
+    return 0;
+}
+
+int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+{
+    *priv_data = *old_priv_data;
+
+    return 0;
+}
+
+void unload(ErlNifEnv* env, void* priv_data)
+{
+}
+
+static ErlNifFunc nif_funcs[] = {
+    NIF_FUNCTIONS(NIF_FUNCTION_ARRAY)
+};
+
+ERL_NIF_INIT(lz4_nif, nif_funcs, load, NULL, upgrade, unload)

c_src/lz4_erlang.h

@@ -0,0 +1,58 @@
+// Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+//
+// This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
+// Public License 1.1 ("MPL") and the Apache License version 2
+// ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+// please see LICENSE-APACHE2.
+//
+// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+// either express or implied. See the LICENSE file for specific language governing
+// rights and limitations of this software.
+//
+// If you have any questions regarding licensing, please contact us at
+// info@rabbitmq.com.
+
+#ifndef __LZ4_ERLANG_H__
+#define __LZ4_ERLANG_H__
+
+#include <lz4frame.h>
+
+// List of atoms used by this NIF.
+//
+// @todo We don't use threads so perhaps we should make nif_helpers
+// better by splitting concerns into threads/not and have nif_helpers
+// decide when to create the _nif_thread_ret atom or not.
+
+#define NIF_ATOMS(A) \
+    A(ok) \
+    A(done) \
+    A(enomem) \
+    A(_nif_thread_ret_)
+
+// List of resources used by this NIF.
+
+#define NIF_RESOURCES(R) \
+    R(LZ4F_cctx) \
+    R(LZ4F_dctx)
+
+// List of functions defined in this NIF.
+
+#define NIF_FUNCTION_NAME(f) lz4_erlang_ ## f
+#define NIF_FUNCTIONS(F) \
+    F(lz4f_compress_frame, 2) \
+    F(lz4f_create_compression_context, 0) \
+    F(lz4f_compress_begin, 2) \
+    F(lz4f_compress_update, 2) \
+    F(lz4f_flush, 1) \
+    F(lz4f_compress_end, 1) \
+    F(lz4f_create_decompression_context, 0) \
+    F(lz4f_get_frame_info, 2) \
+    F(lz4f_decompress, 2)
+
+#include "nif_helpers.h"
+
+NIF_ATOMS(NIF_ATOM_H_DECL)
+NIF_RESOURCES(NIF_RES_H_DECL)
+NIF_FUNCTIONS(NIF_FUNCTION_H_DECL)
+
+#endif

c_src/lz4f.c

@@ -0,0 +1,286 @@
+// Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+//
+// This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
+// Public License 1.1 ("MPL") and the Apache License version 2
+// ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+// please see LICENSE-APACHE2.
+//
+// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+// either express or implied. See the LICENSE file for specific language governing
+// rights and limitations of this software.
+//
+// If you have any questions regarding licensing, please contact us at
+// info@rabbitmq.com.
+
+#include "lz4_erlang.h"
+#include <string.h>
+
+void dtor_LZ4F_cctx(ErlNifEnv* env, void* obj)
+{
+    LZ4F_freeCompressionContext(NIF_RES_GET(LZ4F_cctx, obj));
+}
+
+void dtor_LZ4F_dctx(ErlNifEnv* env, void* obj)
+{
+    LZ4F_freeDecompressionContext(NIF_RES_GET(LZ4F_dctx, obj));
+}
+
+NIF_FUNCTION(lz4f_compress_frame)
+{
+    LZ4F_preferences_t preferences;
+    size_t dstCapacity, dstSize;
+    ErlNifBinary srcBin, dstBin;
+
+    BADARG_IF(!enif_inspect_binary(env, argv[0], &srcBin));
+
+    memset(&preferences, 0, sizeof(preferences));
+
+    // @todo prefs
+
+    dstCapacity = LZ4F_compressFrameBound(srcBin.size, &preferences);
+
+    if (!enif_alloc_binary(dstCapacity, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    dstSize = LZ4F_compressFrame(dstBin.data, dstCapacity, srcBin.data, srcBin.size, &preferences);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_create_compression_context)
+{
+    LZ4F_cctx* cctx;
+    LZ4F_errorCode_t result;
+    ERL_NIF_TERM term;
+
+    result = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
+
+    if (LZ4F_isError(result))
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(result)));
+
+    NIF_RES_TO_TERM(LZ4F_cctx, cctx, term);
+
+    return term;
+}
+
+NIF_FUNCTION(lz4f_compress_begin)
+{
+    void* cctx_res;
+    LZ4F_preferences_t preferences;
+    size_t dstSize;
+    ErlNifBinary dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res));
+
+    memset(&preferences, 0, sizeof(preferences));
+
+    // @todo prefs
+
+    if (!enif_alloc_binary(LZ4F_HEADER_SIZE_MAX, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    dstSize = LZ4F_compressBegin(NIF_RES_GET(LZ4F_cctx, cctx_res),
+        dstBin.data, LZ4F_HEADER_SIZE_MAX, &preferences);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_compress_update)
+{
+    void* cctx_res;
+    size_t dstCapacity, dstSize;
+    ErlNifBinary srcBin, dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res));
+    BADARG_IF(!enif_inspect_binary(env, argv[1], &srcBin));
+
+    // @todo We pass NULL because we don't currently keep the preferences
+    // setup when the user began the compression. It might be done later
+    // as an optimization.
+    dstCapacity = LZ4F_compressBound(srcBin.size, NULL);
+
+    if (!enif_alloc_binary(dstCapacity, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    // We pass NULL because we can't guarantee that the source binary
+    // data will remain for future calls. It may be garbage collected.
+    dstSize = LZ4F_compressUpdate(NIF_RES_GET(LZ4F_cctx, cctx_res),
+        dstBin.data, dstCapacity, srcBin.data, srcBin.size, NULL);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_flush)
+{
+    void* cctx_res;
+    size_t dstCapacity, dstSize;
+    ErlNifBinary dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res));
+
+    // We pass 0 to get the upper bound for this operation.
+    //
+    // @todo We pass NULL because we don't currently keep the preferences
+    // setup when the user began the compression. It might be done later
+    // as an optimization.
+    dstCapacity = LZ4F_compressBound(0, NULL);
+
+    if (!enif_alloc_binary(dstCapacity, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    // We pass NULL because we can't guarantee that the source binary
+    // data will remain for future calls. It may be garbage collected.
+    dstSize = LZ4F_flush(NIF_RES_GET(LZ4F_cctx, cctx_res),
+        dstBin.data, dstCapacity, NULL);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_compress_end)
+{
+    void* cctx_res;
+    size_t dstCapacity, dstSize;
+    ErlNifBinary dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res));
+
+    // We pass 0 to get the upper bound for this operation.
+    //
+    // @todo We pass NULL because we don't currently keep the preferences
+    // setup when the user began the compression. It might be done later
+    // as an optimization.
+    dstCapacity = LZ4F_compressBound(0, NULL);
+
+    if (!enif_alloc_binary(dstCapacity, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    // We pass NULL because we can't guarantee that the source binary
+    // data will remain for future calls. It may be garbage collected.
+    dstSize = LZ4F_compressEnd(NIF_RES_GET(LZ4F_cctx, cctx_res),
+        dstBin.data, dstCapacity, NULL);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_create_decompression_context)
+{
+    LZ4F_dctx* dctx;
+    LZ4F_errorCode_t result;
+    ERL_NIF_TERM term;
+
+    result = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+
+    if (LZ4F_isError(result))
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(result)));
+
+    NIF_RES_TO_TERM(LZ4F_dctx, dctx, term);
+
+    return term;
+}
+
+NIF_FUNCTION(lz4f_get_frame_info)
+{
+    // @todo
+
+    return atom_ok;
+}
+
+NIF_FUNCTION(lz4f_decompress)
+{
+    void* dctx_res;
+    ERL_NIF_TERM head, reversed;
+    size_t dstSize, srcRead, srcSize;
+    ErlNifBinary srcBin, dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_dctx, &dctx_res));
+    BADARG_IF(!enif_inspect_binary(env, argv[1], &srcBin));
+
+    srcRead = 0;
+    srcSize = srcBin.size;
+
+    head = enif_make_list(env, 0);
+
+    while (srcSize) {
+        dstSize = 65536; // Arbitrary maximum size of chunk.
+
+        if (!enif_alloc_binary(dstSize, &dstBin)) {
+            return enif_raise_exception(env, atom_enomem);
+        }
+
+        LZ4F_decompress(NIF_RES_GET(LZ4F_dctx, dctx_res),
+            dstBin.data, &dstSize,
+            srcBin.data + srcRead, &srcSize,
+            NULL);
+
+        if (LZ4F_isError(dstSize)) {
+            enif_release_binary(&dstBin);
+            return enif_raise_exception(env, atom_enomem);
+        }
+
+        if (!enif_realloc_binary(&dstBin, dstSize)) {
+            enif_release_binary(&dstBin);
+            return enif_raise_exception(env, atom_enomem);
+        }
+
+        head = enif_make_list_cell(env, enif_make_binary(env, &dstBin), head);
+
+        srcRead += srcSize;
+        srcSize = srcBin.size - srcRead;
+    }
+
+    // We don't check the return value because no error can occur
+    // according to the function's source code.
+    enif_make_reverse_list(env, head, &reversed);
+
+    return reversed;
+}

ebin/lz4.app

@@ -0,0 +1,8 @@
+{application, 'lz4', [
+	{description, "New project"},
+	{vsn, "0.1.0"},
+	{modules, ['leveled_app','leveled_bookie','leveled_cdb','leveled_codec','leveled_iclerk','leveled_imanifest','leveled_inker','leveled_log','leveled_pclerk','leveled_penciller','leveled_pmanifest','leveled_pmem','leveled_rand','leveled_runner','leveled_sst','leveled_sup','leveled_tictac','leveled_tinybloom','leveled_tree','lz4_nif','lz4f']},
+	{registered, []},
+	{applications, [kernel,stdlib]},
+	{env, []}
+]}.

lz4.d

@@ -0,0 +1,2 @@
+
+COMPILE_FIRST +=

rebar.config

@@ -9,3 +9,10 @@
     {erl_opts, [debug_info, {parse_transform, lager_transform}, {parse_transform, eqc_cover}]},
     {plugins, [rebar_eqc]}]}
  ]}.
+
+{pre_hooks, [
+    {"(linux|darwin|solaris|win32)", clean, "make clean"},
+    {"(freebsd|netbsd|openbsd)", clean, "gmake clean"},
+    {"(linux|darwin|solaris|win32)", compile, "make"},
+    {"(freebsd|netbsd|openbsd)", compile, "gmake"}
+]}.

src/leveled_cdb.erl

@@ -1385,14 +1385,13 @@ perform_write_hash_tables(Handle, HashTreeBin, StartPos) ->
 %% The List passed in should be made up of {Index, Position, Count} tuples
 write_top_index_table(Handle, BasePos, IndexList) ->
     FnWriteIndex = fun({_Index, Pos, Count}, {AccBin, CurrPos}) ->
-        case Count == 0 of
-            true ->
-                PosLE = endian_flip(CurrPos),
-                NextPos = CurrPos;
-            false ->
-                PosLE = endian_flip(Pos),
-                NextPos = Pos + (Count * ?DWORD_SIZE)
-        end, 
+        {PosLE, NextPos} = 
+            case Count == 0 of
+                true ->
+                    {endian_flip(CurrPos), CurrPos};
+                false ->
+                    {endian_flip(Pos), Pos + (Count * ?DWORD_SIZE)}
+            end, 
         CountLE = endian_flip(Count),
         {<<AccBin/binary, PosLE:32, CountLE:32>>, NextPos}
     end,

src/lz4_nif.erl

@@ -0,0 +1,64 @@
+%% Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+%%
+%% This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
+%% Public License 1.1 ("MPL") and the Apache License version 2
+%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+%% please see LICENSE-APACHE2.
+%%
+%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+%% either express or implied. See the LICENSE file for specific language governing
+%% rights and limitations of this software.
+%%
+%% If you have any questions regarding licensing, please contact us at
+%% info@rabbitmq.com.
+
+-module(lz4_nif).
+
+%% lz4f.
+-export([lz4f_compress_frame/2]).
+-export([lz4f_create_compression_context/0]).
+-export([lz4f_compress_begin/2]).
+-export([lz4f_compress_update/2]).
+-export([lz4f_flush/1]).
+-export([lz4f_compress_end/1]).
+-export([lz4f_create_decompression_context/0]).
+-export([lz4f_get_frame_info/2]).
+-export([lz4f_decompress/2]).
+
+-on_load(on_load/0).
+on_load() ->
+    case code:priv_dir(leveled) of
+        {error, _} ->
+            {error, {load_failed, "Could not determine the leveled priv/ directory."}};
+        Path ->
+            erlang:load_nif(filename:join(Path, atom_to_list(?MODULE)), 0)
+    end.
+
+%% lz4f.
+
+lz4f_compress_frame(_, _) ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
+lz4f_create_compression_context() ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
+lz4f_compress_begin(_, _) ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
+lz4f_compress_update(_, _) ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
+lz4f_flush(_) ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
+lz4f_compress_end(_) ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
+lz4f_create_decompression_context() ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
+lz4f_get_frame_info(_, _) ->
+    erlang:nif_error({not_loaded, ?MODULE}).
+
+lz4f_decompress(_, _) ->
+    erlang:nif_error({not_loaded, ?MODULE}).

src/lz4f.erl

@@ -0,0 +1,107 @@
+%% Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+%%
+%% This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
+%% Public License 1.1 ("MPL") and the Apache License version 2
+%% ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+%% please see LICENSE-APACHE2.
+%%
+%% This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+%% either express or implied. See the LICENSE file for specific language governing
+%% rights and limitations of this software.
+%%
+%% If you have any questions regarding licensing, please contact us at
+%% info@rabbitmq.com.
+
+-module(lz4f).
+
+%% Simple compression.
+-export([compress_frame/1]).
+-export([compress_frame/2]).
+
+%% Advanced compression.
+-export([create_compression_context/0]).
+-export([compress_begin/1]).
+-export([compress_begin/2]).
+-export([compress_update/2]).
+-export([flush/1]).
+-export([compress_end/1]).
+
+%% Decompression.
+-export([create_decompression_context/0]).
+-export([get_frame_info/2]).
+-export([decompress/1]).
+-export([decompress/2]).
+
+-type block_size_id() :: default | max64KB | max256KB | max1MB | max4MB.
+-type block_mode() :: linked | independent.
+-type frame_type() :: frame | skippable_frame.
+
+-type frame_info() :: #{
+    block_size_id => block_size_id(),
+    block_mode => block_mode(),
+    content_checksum => boolean(),
+    frame_type => frame_type(),
+    content_size => non_neg_integer()
+}.
+
+-type opts() :: #{
+    frame_info => frame_info(),
+    compression_level => 0..16,
+    auto_flush => boolean()
+}.
+
+-opaque cctx() :: <<>>. %% Resource.
+-export_type([cctx/0]).
+
+-opaque dctx() :: <<>>. %% Resource.
+-export_type([dctx/0]).
+
+-spec compress_frame(binary()) -> binary().
+compress_frame(Data) ->
+    lz4_nif:lz4f_compress_frame(Data, #{}).
+
+-spec compress_frame(binary(), opts()) -> binary().
+compress_frame(Data, Opts) ->
+    lz4_nif:lz4f_compress_frame(Data, Opts).
+
+-spec create_compression_context() -> cctx().
+create_compression_context() ->
+    lz4_nif:lz4f_create_compression_context().
+
+-spec compress_begin(cctx()) -> binary().
+compress_begin(Cctx) ->
+    compress_begin(Cctx, #{}).
+
+-spec compress_begin(cctx(), opts()) -> binary().
+compress_begin(Cctx, Opts) ->
+    lz4_nif:lz4f_compress_begin(Cctx, Opts).
+
+-spec compress_update(cctx(), binary()) -> binary().
+compress_update(Cctx, Data) ->
+    lz4_nif:lz4f_compress_update(Cctx, Data).
+
+-spec flush(cctx()) -> binary().
+flush(Cctx) ->
+    lz4_nif:lz4f_flush(Cctx).
+
+-spec compress_end(cctx()) -> binary().
+compress_end(Cctx) ->
+    lz4_nif:lz4f_compress_end(Cctx).
+
+-spec create_decompression_context() -> dctx().
+create_decompression_context() ->
+    lz4_nif:lz4f_create_decompression_context().
+
+-spec get_frame_info(dctx(), binary()) -> {ok, frame_info(), non_neg_integer()}.
+get_frame_info(Dctx, Data) ->
+    lz4_nif:lz4f_get_frame_info(Dctx, Data).
+
+-spec decompress(binary()) -> iolist().
+decompress(Data) ->
+    decompress(create_decompression_context(), Data).
+
+-spec decompress(dctx(), binary()) -> iolist().
+decompress(Dctx, Data) ->
+    lz4_nif:lz4f_decompress(Dctx, Data).
+
+%% @todo LZ4F_resetDecompressionContext

test/lz4f_SUITE.erl

@@ -0,0 +1,82 @@
+-module(lz4f_SUITE).
+-compile(export_all).
+
+-import(ct_helper, [config/2]).
+-import(ct_helper, [doc/1]).
+
+%% ct.
+
+all() ->
+    [{group, all}].
+
+groups() ->
+    [{all, [parallel], ct_helper:all(?MODULE)}].
+
+init_per_suite(Config) ->
+    [{test_file, config(data_dir, Config) ++ "/pdf_reference_1-7.pdf"}|Config].
+
+end_per_suite(_) ->
+    ok.
+
+%% Tests.
+
+compress_frame1(Config) ->
+    doc("Use lz4f:compress_frame/1 and then decompress back."),
+    {ok, File} = file:read_file(config(test_file, Config)),
+    Compressed = lz4f:compress_frame(File),
+    File = iolist_to_binary(lz4f:decompress(Compressed)),
+    ok.
+
+compress_update(Config) ->
+    doc("Stream compress with lz4f:compress_update/2 and then decompress back."),
+    {ok, File} = file:read_file(config(test_file, Config)),
+    Chunks = do_slice(File),
+    Ctx = lz4f:create_compression_context(),
+    Begin = lz4f:compress_begin(Ctx),
+    CompressedChunks = [lz4f:compress_update(Ctx, C) || C <- Chunks],
+    End = lz4f:compress_end(Ctx),
+    Compressed = iolist_to_binary([Begin, CompressedChunks, End]),
+    File = iolist_to_binary(lz4f:decompress(Compressed)),
+    ok.
+
+compress_flush(Config) ->
+    doc("Stream compress with some lz4f:flush/1 and then decompress back."),
+    {ok, File} = file:read_file(config(test_file, Config)),
+    Chunks = do_insert_flush(do_slice(File)),
+    Ctx = lz4f:create_compression_context(),
+    Begin = lz4f:compress_begin(Ctx),
+    CompressedChunks = [case C of
+        flush -> lz4f:flush(Ctx);
+        _ -> lz4f:compress_update(Ctx, C)
+    end || C <- Chunks],
+    End = lz4f:compress_end(Ctx),
+    Compressed = iolist_to_binary([Begin, CompressedChunks, End]),
+    File = iolist_to_binary(lz4f:decompress(Compressed)),
+    ok.
+
+decompress(Config) ->
+    doc("Compress and then stream decompress with lz4f:decompress/2."),
+    {ok, File} = file:read_file(config(test_file, Config)),
+    Compressed = lz4f:compress_frame(File),
+    Chunks = do_slice(Compressed),
+    Ctx = lz4f:create_decompression_context(),
+    DecompressedChunks = [lz4f:decompress(Ctx, C) || C <- Chunks],
+    File = iolist_to_binary(DecompressedChunks),
+    ok.
+
+%% Internal.
+
+do_insert_flush(L) ->
+    do_insert_flush(L, 0).
+
+do_insert_flush([], _) ->
+    [];
+do_insert_flush(L, 5) ->
+    [flush|do_insert_flush(L, 0)];
+do_insert_flush([H|T], N) ->
+    [H|do_insert_flush(T, N + 1)].
+
+do_slice(<<Bin:1000000/binary, R/bits>>) ->
+    [Bin|do_slice(R)];
+do_slice(Bin) ->
+    [Bin].