Failed attempt to hack in LZ4

c_src/env.mk

@@ -0,0 +1,3 @@
+ERTS_INCLUDE_DIR ?= /Users/martinsumner/erlang/18.3/erts-7.3/include/
+ERL_INTERFACE_INCLUDE_DIR ?= /Users/martinsumner/erlang/18.3/lib/erl_interface-3.8.2/include
+ERL_INTERFACE_LIB_DIR ?= /Users/martinsumner/erlang/18.3/lib/erl_interface-3.8.2/lib

c_src/lz4.c

@@ -0,0 +1,47 @@
+// Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+//
+// This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
+// Public License 1.1 ("MPL") and the Apache License version 2
+// ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+// please see LICENSE-APACHE2.
+//
+// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+// either express or implied. See the LICENSE file for specific language governing
+// rights and limitations of this software.
+//
+// If you have any questions regarding licensing, please contact us at
+// info@rabbitmq.com.
+
+#include "lz4_erlang.h"
+
+NIF_ATOMS(NIF_ATOM_DECL)
+NIF_RESOURCES(NIF_RES_DECL)
+
+int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info);
+int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info);
+void unload(ErlNifEnv* env, void* priv_data);
+
+int load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info)
+{
+    NIF_ATOMS(NIF_ATOM_INIT)
+    NIF_RESOURCES(NIF_RES_INIT)
+
+    return 0;
+}
+
+int upgrade(ErlNifEnv* env, void** priv_data, void** old_priv_data, ERL_NIF_TERM load_info)
+{
+    *priv_data = *old_priv_data;
+
+    return 0;
+}
+
+void unload(ErlNifEnv* env, void* priv_data)
+{
+}
+
+static ErlNifFunc nif_funcs[] = {
+    NIF_FUNCTIONS(NIF_FUNCTION_ARRAY)
+};
+
+ERL_NIF_INIT(lz4_nif, nif_funcs, load, NULL, upgrade, unload)

c_src/lz4_erlang.h

@@ -0,0 +1,58 @@
+// Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+//
+// This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
+// Public License 1.1 ("MPL") and the Apache License version 2
+// ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+// please see LICENSE-APACHE2.
+//
+// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+// either express or implied. See the LICENSE file for specific language governing
+// rights and limitations of this software.
+//
+// If you have any questions regarding licensing, please contact us at
+// info@rabbitmq.com.
+
+#ifndef __LZ4_ERLANG_H__
+#define __LZ4_ERLANG_H__
+
+#include <lz4frame.h>
+
+// List of atoms used by this NIF.
+//
+// @todo We don't use threads so perhaps we should make nif_helpers
+// better by splitting concerns into threads/not and have nif_helpers
+// decide when to create the _nif_thread_ret atom or not.
+
+#define NIF_ATOMS(A) \
+    A(ok) \
+    A(done) \
+    A(enomem) \
+    A(_nif_thread_ret_)
+
+// List of resources used by this NIF.
+
+#define NIF_RESOURCES(R) \
+    R(LZ4F_cctx) \
+    R(LZ4F_dctx)
+
+// List of functions defined in this NIF.
+
+#define NIF_FUNCTION_NAME(f) lz4_erlang_ ## f
+#define NIF_FUNCTIONS(F) \
+    F(lz4f_compress_frame, 2) \
+    F(lz4f_create_compression_context, 0) \
+    F(lz4f_compress_begin, 2) \
+    F(lz4f_compress_update, 2) \
+    F(lz4f_flush, 1) \
+    F(lz4f_compress_end, 1) \
+    F(lz4f_create_decompression_context, 0) \
+    F(lz4f_get_frame_info, 2) \
+    F(lz4f_decompress, 2)
+
+#include "nif_helpers.h"
+
+NIF_ATOMS(NIF_ATOM_H_DECL)
+NIF_RESOURCES(NIF_RES_H_DECL)
+NIF_FUNCTIONS(NIF_FUNCTION_H_DECL)
+
+#endif

c_src/lz4f.c

@@ -0,0 +1,286 @@
+// Copyright (c) 2017-Present Pivotal Software, Inc.  All rights reserved.
+//
+// This package, the LZ4 binding for Erlang, is double-licensed under the Mozilla
+// Public License 1.1 ("MPL") and the Apache License version 2
+// ("ASL"). For the MPL, please see LICENSE-MPL-RabbitMQ. For the ASL,
+// please see LICENSE-APACHE2.
+//
+// This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND,
+// either express or implied. See the LICENSE file for specific language governing
+// rights and limitations of this software.
+//
+// If you have any questions regarding licensing, please contact us at
+// info@rabbitmq.com.
+
+#include "lz4_erlang.h"
+#include <string.h>
+
+void dtor_LZ4F_cctx(ErlNifEnv* env, void* obj)
+{
+    LZ4F_freeCompressionContext(NIF_RES_GET(LZ4F_cctx, obj));
+}
+
+void dtor_LZ4F_dctx(ErlNifEnv* env, void* obj)
+{
+    LZ4F_freeDecompressionContext(NIF_RES_GET(LZ4F_dctx, obj));
+}
+
+NIF_FUNCTION(lz4f_compress_frame)
+{
+    LZ4F_preferences_t preferences;
+    size_t dstCapacity, dstSize;
+    ErlNifBinary srcBin, dstBin;
+
+    BADARG_IF(!enif_inspect_binary(env, argv[0], &srcBin));
+
+    memset(&preferences, 0, sizeof(preferences));
+
+    // @todo prefs
+
+    dstCapacity = LZ4F_compressFrameBound(srcBin.size, &preferences);
+
+    if (!enif_alloc_binary(dstCapacity, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    dstSize = LZ4F_compressFrame(dstBin.data, dstCapacity, srcBin.data, srcBin.size, &preferences);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_create_compression_context)
+{
+    LZ4F_cctx* cctx;
+    LZ4F_errorCode_t result;
+    ERL_NIF_TERM term;
+
+    result = LZ4F_createCompressionContext(&cctx, LZ4F_VERSION);
+
+    if (LZ4F_isError(result))
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(result)));
+
+    NIF_RES_TO_TERM(LZ4F_cctx, cctx, term);
+
+    return term;
+}
+
+NIF_FUNCTION(lz4f_compress_begin)
+{
+    void* cctx_res;
+    LZ4F_preferences_t preferences;
+    size_t dstSize;
+    ErlNifBinary dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res));
+
+    memset(&preferences, 0, sizeof(preferences));
+
+    // @todo prefs
+
+    if (!enif_alloc_binary(LZ4F_HEADER_SIZE_MAX, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    dstSize = LZ4F_compressBegin(NIF_RES_GET(LZ4F_cctx, cctx_res),
+        dstBin.data, LZ4F_HEADER_SIZE_MAX, &preferences);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_compress_update)
+{
+    void* cctx_res;
+    size_t dstCapacity, dstSize;
+    ErlNifBinary srcBin, dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res));
+    BADARG_IF(!enif_inspect_binary(env, argv[1], &srcBin));
+
+    // @todo We pass NULL because we don't currently keep the preferences
+    // setup when the user began the compression. It might be done later
+    // as an optimization.
+    dstCapacity = LZ4F_compressBound(srcBin.size, NULL);
+
+    if (!enif_alloc_binary(dstCapacity, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    // We pass NULL because we can't guarantee that the source binary
+    // data will remain for future calls. It may be garbage collected.
+    dstSize = LZ4F_compressUpdate(NIF_RES_GET(LZ4F_cctx, cctx_res),
+        dstBin.data, dstCapacity, srcBin.data, srcBin.size, NULL);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_flush)
+{
+    void* cctx_res;
+    size_t dstCapacity, dstSize;
+    ErlNifBinary dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res));
+
+    // We pass 0 to get the upper bound for this operation.
+    //
+    // @todo We pass NULL because we don't currently keep the preferences
+    // setup when the user began the compression. It might be done later
+    // as an optimization.
+    dstCapacity = LZ4F_compressBound(0, NULL);
+
+    if (!enif_alloc_binary(dstCapacity, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    // We pass NULL because we can't guarantee that the source binary
+    // data will remain for future calls. It may be garbage collected.
+    dstSize = LZ4F_flush(NIF_RES_GET(LZ4F_cctx, cctx_res),
+        dstBin.data, dstCapacity, NULL);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_compress_end)
+{
+    void* cctx_res;
+    size_t dstCapacity, dstSize;
+    ErlNifBinary dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_cctx, &cctx_res));
+
+    // We pass 0 to get the upper bound for this operation.
+    //
+    // @todo We pass NULL because we don't currently keep the preferences
+    // setup when the user began the compression. It might be done later
+    // as an optimization.
+    dstCapacity = LZ4F_compressBound(0, NULL);
+
+    if (!enif_alloc_binary(dstCapacity, &dstBin))
+        return enif_raise_exception(env, atom_enomem);
+
+    // We pass NULL because we can't guarantee that the source binary
+    // data will remain for future calls. It may be garbage collected.
+    dstSize = LZ4F_compressEnd(NIF_RES_GET(LZ4F_cctx, cctx_res),
+        dstBin.data, dstCapacity, NULL);
+
+    if (LZ4F_isError(dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(dstSize)));
+    }
+
+    if (!enif_realloc_binary(&dstBin, dstSize)) {
+        enif_release_binary(&dstBin);
+        return enif_raise_exception(env, atom_enomem);
+    }
+
+    return enif_make_binary(env, &dstBin);
+}
+
+NIF_FUNCTION(lz4f_create_decompression_context)
+{
+    LZ4F_dctx* dctx;
+    LZ4F_errorCode_t result;
+    ERL_NIF_TERM term;
+
+    result = LZ4F_createDecompressionContext(&dctx, LZ4F_VERSION);
+
+    if (LZ4F_isError(result))
+        return enif_raise_exception(env, enif_make_atom(env, LZ4F_getErrorName(result)));
+
+    NIF_RES_TO_TERM(LZ4F_dctx, dctx, term);
+
+    return term;
+}
+
+NIF_FUNCTION(lz4f_get_frame_info)
+{
+    // @todo
+
+    return atom_ok;
+}
+
+NIF_FUNCTION(lz4f_decompress)
+{
+    void* dctx_res;
+    ERL_NIF_TERM head, reversed;
+    size_t dstSize, srcRead, srcSize;
+    ErlNifBinary srcBin, dstBin;
+
+    BADARG_IF(!enif_get_resource(env, argv[0], res_LZ4F_dctx, &dctx_res));
+    BADARG_IF(!enif_inspect_binary(env, argv[1], &srcBin));
+
+    srcRead = 0;
+    srcSize = srcBin.size;
+
+    head = enif_make_list(env, 0);
+
+    while (srcSize) {
+        dstSize = 65536; // Arbitrary maximum size of chunk.
+
+        if (!enif_alloc_binary(dstSize, &dstBin)) {
+            return enif_raise_exception(env, atom_enomem);
+        }
+
+        LZ4F_decompress(NIF_RES_GET(LZ4F_dctx, dctx_res),
+            dstBin.data, &dstSize,
+            srcBin.data + srcRead, &srcSize,
+            NULL);
+
+        if (LZ4F_isError(dstSize)) {
+            enif_release_binary(&dstBin);
+            return enif_raise_exception(env, atom_enomem);
+        }
+
+        if (!enif_realloc_binary(&dstBin, dstSize)) {
+            enif_release_binary(&dstBin);
+            return enif_raise_exception(env, atom_enomem);
+        }
+
+        head = enif_make_list_cell(env, enif_make_binary(env, &dstBin), head);
+
+        srcRead += srcSize;
+        srcSize = srcBin.size - srcRead;
+    }
+
+    // We don't check the return value because no error can occur
+    // according to the function's source code.
+    enif_make_reverse_list(env, head, &reversed);
+
+    return reversed;
+}