diff --git a/Makefile b/Makefile index 933ce28..ab8b307 100644 --- a/Makefile +++ b/Makefile @@ -14,9 +14,10 @@ endif ERLWARE_COMMONS_PLT=$(CURDIR)/.erlware_commons_plt -.PHONY: all compile doc clean test dialyzer typer shell distclean pdf get-deps escript +.PHONY: all compile doc clean test dialyzer typer shell distclean pdf get-deps \ + rebuild -all: compile test dialyzer +all: compile dialyzer doc test get-deps: $(REBAR) get-deps @@ -34,11 +35,11 @@ test: compile $(ERLWARE_COMMONS_PLT): @echo Building local plt at $(ERLWARE_COMMONS_PLT) @echo - - dialyzer --output_plt $(ERLWARE_COMMONS_PLT) --build_plt \ + - dialyzer --fullpath --output_plt $(ERLWARE_COMMONS_PLT) --build_plt \ --apps erts kernel stdlib eunit -r deps dialyzer: $(ERLWARE_COMMONS_PLT) - dialyzer --plt $(ERLWARE_COMMONS_PLT) -Wrace_conditions --src src + dialyzer --fullpath --plt $(ERLWARE_COMMONS_PLT) -Wrace_conditions --src src typer: typer --plt $(ERLWARE_COMMONS_PLT) -r ./src @@ -62,3 +63,5 @@ clean: distclean: clean rm -rf $(ERLWARE_COMMONS_PLT) rm -rvf $(CURDIR)/deps/* + +rebuild: distclean all diff --git a/README.md b/README.md index f2cab01..3a8ed83 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,107 @@ Goals for the project * High Quality * Well Documented * Well Tested + +Currently Available Modules/Systems +------------------------------------ + +### [ec_date](https://github.com/erlware/erlware_commons/blob/master/src/ec_date.erl) + +This module formats erlang dates in the form {{Year, Month, Day}, +{Hour, Minute, Second}} to printable strings, using (almost) +equivalent formatting rules as http://uk.php.net/date, US vs European +dates are disambiguated in the same way as +http://uk.php.net/manual/en/function.strtotime.php That is, Dates in +the m/d/y or d-m-y formats are disambiguated by looking at the +separator between the various components: if the separator is a slash +(/), then the American m/d/y is assumed; whereas if the separator is a +dash (-) or a dot (.), then the European d-m-y format is assumed. To +avoid potential ambiguity, it's best to use ISO 8601 (YYYY-MM-DD) +dates. + +erlang has no concept of timezone so the following formats are not +implemented: B e I O P T Z formats c and r will also differ slightly + +### [ec_file](https://github.com/erlware/erlware_commons/blob/master/src/ec_file.erl) + +A set of commonly defined helper functions for files that are not +included in stdlib. + +### [ec_plists](https://github.com/erlware/erlware_commons/blob/master/src/ec_plists.erl) + +plists is a drop-in replacement for module lists, making most +list operations parallel. It can operate on each element in parallel, +for IO-bound operations, on sublists in parallel, for taking advantage +of multi-core machines with CPU-bound operations, and across erlang +nodes, for parallizing inside a cluster. It handles errors and node +failures. It can be configured, tuned, and tweaked to get optimal +performance while minimizing overhead. + +Almost all the functions are identical to equivalent functions in +lists, returning exactly the same result, and having both a form with +an identical syntax that operates on each element in parallel and a +form which takes an optional "malt", a specification for how to +parallize the operation. + +fold is the one exception, parallel fold is different from linear +fold. This module also include a simple mapreduce implementation, and +the function runmany. All the other functions are implemented with +runmany, which is as a generalization of parallel list operations. + +### [ec_semver](https://github.com/erlware/erlware_commons/blob/master/src/ec_semver.erl) + +A complete parser for the [semver](http://semver.org/) +standard. Including a complete set of conforming comparison functions. + +### [ec_lists](https://github.com/ericbmerritt/erlware_commons/blob/master/src/ec_lists.erl) + +A set of additional list manipulation functions designed to supliment +the `lists` module in stdlib. + +### [ec_talk](https://github.com/erlware/erlware_commons/blob/master/src/ec_talk.erl) + +A set of simple utility functions to falicitate command line +communication with a user. + +Signatures +----------- + +Other languages, have built in support for **Interface** or +**signature** functionality. Java has Interfaces, SML has +Signatures. Erlang, though, doesn't currently support this model, at +least not directly. There are a few ways you can approximate it. We +have defined a mechnism called *signatures* and several modules that +to serve as examples and provide a good set of *dictionary* +signatures. More information about signatures can be found at +[signature](https://github.com/erlware/erlware_commons/blob/master/doc/signatures.md). + + +### [ec_dictionary](https://github.com/erlware/erlware_commons/blob/master/src/ec_dictionary.erl) + +A signature that supports association of keys to values. A map cannot +contain duplicate keys; each key can map to at most one value. + +### [ec_dict](https://github.com/erlware/erlware_commons/blob/master/src/ec_dict.erl) + +This provides an implementation of the ec_dictionary signature using +erlang's dicts as a base. The function documentation for ec_dictionary +applies here as well. + +### [ec_gb_trees](https://github.com/ericbmerritt/erlware_commons/blob/master/src/ec_gb_trees.erl) + +This provides an implementation of the ec_dictionary signature using +erlang's gb_trees as a base. The function documentation for +ec_dictionary applies here as well. + +### [ec_orddict](https://github.com/ericbmerritt/erlware_commons/blob/master/src/ec_orddict.erl) + +This provides an implementation of the ec_dictionary signature using +erlang's orddict as a base. The function documentation for +ec_dictionary applies here as well. + +### [ec_rbdict](https://github.com/ericbmerritt/erlware_commons/blob/master/src/ec_rbdict.erl) + +This provides an implementation of the ec_dictionary signature using +Robert Virding's rbdict module as a base. The function documentation +for ec_dictionary applies here as well. diff --git a/rebar.config b/rebar.config index 6284cba..08863d2 100644 --- a/rebar.config +++ b/rebar.config @@ -1,12 +1,25 @@ %% -*- mode: Erlang; fill-column: 80; comment-column: 75; -*- -%% These are all only compile time dependencies +%% Dependencies ================================================================ {deps, [{neotoma, "", - {git, "https://github.com/seancribbs/neotoma.git", {tag, "1.5"}}}, - {proper, "", {git, "https://github.com/manopapad/proper.git", {branch, master}}}]}. + {git, "https://github.com/seancribbs/neotoma.git", {branch, master}}}, + {proper, "", {git, "https://github.com/manopapad/proper.git", {branch, master}}}, + {rebar_vsn_plugin, ".*", {git, "https://github.com/erlware/rebar_vsn_plugin.git", + {branch, "master"}}}]}. {erl_first_files, ["ec_dictionary"]}. +%% Compiler Options ============================================================ {erl_opts, [debug_info, warnings_as_errors]}. + +%% EUnit ======================================================================= +{eunit_opts, [verbose, + {report, {eunit_surefire, [{dir, "."}]}}]}. + +{cover_enabled, true}. +{cover_print_enabled, true}. + +%% Rebar Plugins ============================================================== +{plugins, [rebar_vsn_plugin]}. diff --git a/src/ec_compile.erl b/src/ec_compile.erl new file mode 100644 index 0000000..482bdb3 --- /dev/null +++ b/src/ec_compile.erl @@ -0,0 +1,107 @@ +%%%------------------------------------------------------------------- +%%% @author Eric Merritt <> +%%% @copyright (C) 2011, Erlware, LLC. +%%% @doc +%%% These are various utility functions to help with compiling and +%%% decompiling erlang source. They are mostly useful to the +%%% language/parse transform implementor. +%%% @end +%%%------------------------------------------------------------------- +-module(ec_compile). + +-export([beam_to_erl_source/2, + erl_source_to_core_ast/1, + erl_source_to_erl_ast/1, + erl_source_to_asm/1, + erl_string_to_core_ast/1, + erl_string_to_erl_ast/1, + erl_string_to_asm/1]). + +%%%=================================================================== +%%% API +%%%=================================================================== + +%% @doc decompile a beam file that has been compiled with +debug_info +%% into a erlang source file +%% +%% @param BeamFName the name of the beamfile +%% @param ErlFName the name of the erlang file where the generated +%% source file will be output. This should *not* be the same as the +%% source file that created the beamfile unless you want to overwrite +%% it. +-spec beam_to_erl_source(string(), string()) -> ok | term(). +beam_to_erl_source(BeamFName, ErlFName) -> + case beam_lib:chunks(BeamFName, [abstract_code]) of + {ok, {_, [{abstract_code, {raw_abstract_v1,Forms}}]}} -> + Src = + erl_prettypr:format(erl_syntax:form_list(tl(Forms))), + {ok, Fd} = file:open(ErlFName, [write]), + io:fwrite(Fd, "~s~n", [Src]), + file:close(Fd); + Error -> + Error + end. + +%% @doc compile an erlang source file into a Core Erlang AST +%% +%% @param Path - The path to the erlang source file +-spec erl_source_to_core_ast(file:filename()) -> CoreAst::term(). +erl_source_to_core_ast(Path) -> + {ok, Contents} = file:read_file(Path), + erl_string_to_core_ast(binary_to_list(Contents)). + +%% @doc compile an erlang source file into an Erlang AST +%% +%% @param Path - The path to the erlang source file +-spec erl_source_to_erl_ast(file:filename()) -> ErlangAst::term(). +erl_source_to_erl_ast(Path) -> + {ok, Contents} = file:read_file(Path), + erl_string_to_erl_ast(binary_to_list(Contents)). + +%% @doc compile an erlang source file into erlang terms that represent +%% the relevant ASM +%% +%% @param Path - The path to the erlang source file +-spec erl_source_to_asm(file:filename()) -> ErlangAsm::term(). +erl_source_to_asm(Path) -> + {ok, Contents} = file:read_file(Path), + erl_string_to_asm(binary_to_list(Contents)). + +%% @doc compile a string representing an erlang expression into an +%% Erlang AST +%% +%% @param StringExpr - The path to the erlang source file +-spec erl_string_to_erl_ast(string()) -> ErlangAst::term(). +erl_string_to_erl_ast(StringExpr) -> + Forms0 = + lists:foldl(fun(<<>>, Acc) -> + Acc; + (<<"\n\n">>, Acc) -> + Acc; + (El, Acc) -> + {ok, Tokens, _} = + erl_scan:string(binary_to_list(El) + ++ "."), + [Tokens | Acc] + end, [], re:split(StringExpr, "\\.\n")), + %% No need to reverse. This will rereverse for us + lists:foldl(fun(Form, Forms) -> + {ok, ErlAST} = erl_parse:parse_form(Form), + [ErlAST | Forms] + end, [], Forms0). + +%% @doc compile a string representing an erlang expression into a +%% Core Erlang AST +%% +%% @param StringExpr - The path to the erlang source file +-spec erl_string_to_core_ast(string()) -> CoreAst::term(). +erl_string_to_core_ast(StringExpr) -> + compile:forms(erl_string_to_erl_ast(StringExpr), [to_core]). + +%% @doc compile a string representing an erlang expression into a term +%% that represents the ASM +%% +%% @param StringExpr - The path to the erlang source file +-spec erl_string_to_asm(string()) -> ErlangAsm::term(). +erl_string_to_asm(StringExpr) -> + compile:forms(erl_string_to_erl_ast(StringExpr), ['S']). diff --git a/src/ec_file.erl b/src/ec_file.erl index c5cbe56..45adb2a 100644 --- a/src/ec_file.erl +++ b/src/ec_file.erl @@ -7,10 +7,12 @@ -module(ec_file). -export([ + exists/1, copy/2, copy/3, insecure_mkdtemp/0, mkdir_path/1, + mkdir_p/1, find/2, is_symlink/1, remove/1, @@ -35,12 +37,21 @@ %% Types %%============================================================================ -type option() :: recursive. --type void() :: ok. + %%%=================================================================== %%% API %%%=================================================================== +-spec exists(file:filename()) -> boolean(). +exists(Filename) -> + case file:read_file_info(Filename) of + {ok, _} -> + true; + {error, _Reason} -> + false + end. + %% @doc copy an entire directory to another location. --spec copy(file:name(), file:name(), Options::[option()]) -> void(). +-spec copy(file:name(), file:name(), Options::[option()]) -> ok | {error, Reason::term()}. copy(From, To, []) -> copy(From, To); copy(From, To, [recursive] = Options) -> @@ -229,7 +240,7 @@ tmp() -> end. %% Copy the subfiles of the From directory to the to directory. --spec copy_subfiles(file:name(), file:name(), [option()]) -> void(). +-spec copy_subfiles(file:name(), file:name(), [option()]) -> {error, Reason::term()} | ok. copy_subfiles(From, To, Options) -> Fun = fun(ChildFrom) -> @@ -313,6 +324,16 @@ setup_base_and_target() -> ok = file:write_file(NoName, DummyContents), {BaseDir, SourceDir, {Name1, Name2, Name3, NoName}}. +exists_test() -> + BaseDir = insecure_mkdtemp(), + SourceDir = filename:join([BaseDir, "source1"]), + NoName = filename:join([SourceDir, "noname"]), + ok = file:make_dir(SourceDir), + Name1 = filename:join([SourceDir, "fileone"]), + ok = file:write_file(Name1, <<"Testn">>), + ?assertMatch(true, exists(Name1)), + ?assertMatch(false, exists(NoName)). + find_test() -> %% Create a directory in /tmp for the test. Clean everything afterwards {BaseDir, _SourceDir, {Name1, Name2, Name3, _NoName}} = setup_base_and_target(), diff --git a/src/ec_plists.erl b/src/ec_plists.erl index cd14697..a021d02 100644 --- a/src/ec_plists.erl +++ b/src/ec_plists.erl @@ -1,264 +1,942 @@ -%%%------------------------------------------------------------------- +%%% -*- mode: Erlang; fill-column: 80; comment-column: 75; -*- +%%% The MIT License +%%% +%%% Copyright (c) 2007 Stephen Marsh +%%% +%%% Permission is hereby granted, free of charge, to any person obtaining a copy +%%% of this software and associated documentation files (the "Software"), to deal +%%% in the Software without restriction, including without limitation the rights +%%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%%% copies of the Software, and to permit persons to whom the Software is +%%% furnished to do so, subject to the following conditions: +%%% +%%% The above copyright notice and this permission notice shall be included in +%%% all copies or substantial portions of the Software. +%%% +%%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%%% THE SOFTWARE. +%%%--------------------------------------------------------------------------- +%%% @author Stephen Marsh +%%% @copyright 2007 Stephen Marsh freeyourmind ++ [$@|gmail.com] %%% @doc -%%% simple parrallel map. Originally provided by Joe Armstrong -%%% on the erlang questions mailing list. -%%% @end -%%%------------------------------------------------------------------- +%%% plists is a drop-in replacement for module lists, making +%%% most list operations parallel. It can operate on each element in +%%% parallel, for IO-bound operations, on sublists in parallel, for +%%% taking advantage of multi-core machines with CPU-bound operations, +%%% and across erlang nodes, for parallizing inside a cluster. It +%%% handles errors and node failures. It can be configured, tuned, and +%%% tweaked to get optimal performance while minimizing overhead. +%%% +%%% Almost all the functions are identical to equivalent functions in +%%% lists, returning exactly the same result, and having both a form +%%% with an identical syntax that operates on each element in parallel +%%% and a form which takes an optional "malt", a specification for how +%%% to parallize the operation. +%%% +%%% fold is the one exception, parallel fold is different from linear +%%% fold. This module also include a simple mapreduce implementation, +%%% and the function runmany. All the other functions are implemented +%%% with runmany, which is as a generalization of parallel list +%%% operations. +%%% +%%% Malts +%%% ===== +%%% +%%% A malt specifies how to break a list into sublists, and can optionally +%%% specify a timeout, which nodes to run on, and how many processes to start +%%% per node. +%%% +%%% Malt = MaltComponent | [MaltComponent] +%%% MaltComponent = SubListSize::integer() | {processes, integer()} | +%%% {processes, schedulers} | +%%% {timeout, Milliseconds::integer()} | {nodes, [NodeSpec]}
+%%% +%%% NodeSpec = Node::atom() | {Node::atom(), NumProcesses::integer()} | +%%% {Node::atom(), schedulers} +%%% +%%% An integer can be given to specify the exact size for sublists. 1 +%%% is a good choice for IO-bound operations and when the operation on +%%% each list element is expensive. Larger numbers minimize overhead +%%% and are faster for cheap operations. +%%% +%%% If the integer is omitted, and you have specified a `{processes, +%%% X}`, the list is split into X sublists. This is only useful when +%%% the time to process each element is close to identical and you +%%% know exactly how many lines of execution are available to you. +%%% +%%% If neither of the above applies, the sublist size defaults to 1. +%%% +%%% You can use `{processes, X}` to have the list processed by `X` +%%% processes on the local machine. A good choice for `X` is the +%%% number of lines of execution (cores) the machine provides. This +%%% can be done automatically with {processes, schedulers}, which sets +%%% the number of processes to the number of schedulers in the erlang +%%% virtual machine (probably equal to the number of cores). +%%% +%%% `{timeout, Milliseconds}` specifies a timeout. This is a timeout +%%% for the entire operation, both operating on the sublists and +%%% combining the results. exit(timeout) is evaluated if the timeout +%%% is exceeded. +%%% +%%% `{nodes, NodeList}` specifies that the operation should be done +%%% across nodes. Every element of NodeList is of the form +%%% `{NodeName, NumProcesses}` or NodeName, which means the same as +%%% `{NodeName, 1}`. plists runs NumProcesses processes on NodeName +%%% concurrently. A good choice for NumProcesses is the number of +%%% lines of execution (cores) a node provides plus one. This ensures +%%% the node is completely busy even when fetching a new sublist. This +%%% can be done automatically with `{NodeName, schedulers}`, in which +%%% case plists uses a cached value if it has one, and otherwise finds +%%% the number of schedulers in the remote node and adds one. This +%%% will ensure at least one busy process per core (assuming the node +%%% has a scheduler for each core). +%%% +%%% plists is able to recover if a node goes down. If all nodes go +%%% down, exit(allnodescrashed) is evaluated. +%%% +%%% Any of the above may be used as a malt, or may be combined into a +%%% list. `{nodes, NodeList}` and {processes, X} may not be combined. +%%% +%%% Examples +%%% ======== +%%% +%%% %%start a process for each element (1-element sublists)< +%%% 1 +%%% +%%% %% start a process for each ten elements (10-element sublists) +%%% 10 +%%% +%%% %% split the list into two sublists and process in two processes +%%% {processes, 2} +%%% +%%% %% split the list into X sublists and process in X processes, +%%% %% where X is the number of cores in the machine +%%% {processes, schedulers} +%%% +%%% %% split the list into 10-element sublists and process in two processes +%%% [10, {processes, 2}] +%%% +%%% %% timeout after one second. Assumes that a process should be started +%%% %% for each element.
+%%% {timeout, 1000} +%%% +%%% %% Runs 3 processes at a time on apple@desktop, and 2 on orange@laptop +%%% %% This is the best way to utilize all the CPU-power of a dual-core
+%%% %% desktop and a single-core laptop. Assumes that the list should be
+%%% %% split into 1-element sublists.
+%%% {nodes, [{apple@desktop, 3}, {orange@laptop, 2}]} +%%% +%%% %% Like above, but makes plists figure out how many processes to use. +%%% {nodes, [{apple@desktop, schedulers}, {orange@laptop, schedulers}]} +%%% +%%% %% Gives apple and orange three seconds to process the list as
+%%% %% 100-element sublists.
+%%% [100, {timeout, 3000}, {nodes, [{apple@desktop, 3}, {orange@laptop, 2}]}] +%%% +%%% Aside: Why Malt? +%%% ================ +%%% +%%% I needed a word for this concept, so maybe my subconsciousness +%%% gave me one by making me misspell multiply. Maybe it is an acronym +%%% for Malt is A List Tearing Specification. Maybe it is a beer +%%% metaphor, suggesting that code only runs in parallel if bribed +%%% with spirits. It's jargon, learn it or you can't be part of the +%%% in-group. +%%% +%%% Messages and Errors +%%% =================== +%%% +%%% plists assures that no extraneous messages are left in or will +%%% later enter the message queue. This is guaranteed even in the +%%% event of an error. +%%% +%%% Errors in spawned processes are caught and propagated to the +%%% calling process. If you invoke +%%% +%%% plists:map(fun (X) -> 1/X end, [1, 2, 3, 0]). +%%% +%%% you get a badarith error, exactly like when you use lists:map. +%%% +%%% plists uses monitors to watch the processes it spawns. It is not a +%%% good idea to invoke plists when you are already monitoring +%%% processes. If one of them does a non-normal exit, plists receives +%%% the 'DOWN' message believing it to be from one of its own +%%% processes. The error propagation system goes into effect, which +%%% results in the error occuring in the calling process. +%%% -module(ec_plists). --export([map/2, - map/3, - ftmap/2, - ftmap/3, - filter/2, - filter/3]). +-export([all/2, all/3, + any/2, any/3, + filter/2, filter/3, + fold/3, fold/4, fold/5, + foreach/2, foreach/3, + map/2, map/3, + ftmap/2, ftmap/3, + partition/2, partition/3, + sort/1, sort/2, sort/3, + usort/1, usort/2, usort/3, + mapreduce/2, mapreduce/3, mapreduce/5, + runmany/3, runmany/4]). --export_type([thunk/0]). +-export_type([malt/0, malt_component/0, node_spec/0, fuse/0, fuse_fun/0]). -%%============================================================================= -%% Types -%%============================================================================= --type thunk() :: fun((any()) -> any()). +%%============================================================================ +%% types +%%============================================================================ -%%============================================================================= -%% Public API -%%============================================================================= +-type malt() :: malt_component() | [malt_component()]. -%% @doc Takes a function and produces a list of the result of the function -%% applied to each element of the argument list. A timeout is optional. -%% In the event of a timeout or an exception the entire map will fail -%% with an excption with class throw. --spec map(fun(), [any()]) -> [any()]. -map(Fun, List) -> - map(Fun, List, infinity). +-type malt_component() :: SubListSize::integer() + | {processes, integer()} + | {processes, schedulers} + | {timeout, Milliseconds::integer()} + | {nodes, [node_spec()]}. --spec map(thunk(), [any()], timeout() | infinity) -> [any()]. -map(Fun, List, Timeout) -> - run_list_fun_in_parallel(map, Fun, List, Timeout). +-type node_spec() :: Node::atom() + | {Node::atom(), NumProcesses::integer()} + | {Node::atom(), schedulers}. -%% @doc Takes a function and produces a list of the result of the function -%% applied to each element of the argument list. A timeout is optional. -%% This function differes from regular map in that it is fault tolerant. -%% If a timeout or an exception occurs while processing an element in -%% the input list the ftmap operation will continue to function. Timeouts -%% and exceptions will be reflected in the output of this function. -%% All application level results are wrapped in a tuple with the tag -%% 'value'. Exceptions will come through as they are and timeouts will -%% return as the atom timeout. -%% This is useful when the ftmap is being used for side effects. -%% 
-%% 2> ftmap(fun(N) -> factorial(N) end, [1, 2, 1000000, "not num"], 100)
-%% [{value, 1}, {value, 2}, timeout, {badmatch, ...}]
-%%
--spec ftmap(thunk(), [any()]) -> [{value, any()} | any()]. -ftmap(Fun, List) -> - ftmap(Fun, List, infinity). +-type fuse_fun() :: fun((term(), term()) -> term()). +-type fuse() :: fuse_fun() | {recursive, fuse_fun()} | {reverse, fuse_fun()}. +-type el_fun() :: fun((term()) -> term()). --spec ftmap(thunk(), [any()], timeout() | infinity) -> [{value, any()} | any()]. -ftmap(Fun, List, Timeout) -> - run_list_fun_in_parallel(ftmap, Fun, List, Timeout). +%%============================================================================ +%% API +%%============================================================================ -%% @doc Returns a list of the elements in the supplied list which -%% the function Fun returns true. A timeout is optional. In the -%% event of a timeout the filter operation fails. --spec filter(thunk(), [any()]) -> [any()]. -filter(Fun, List) -> - filter(Fun, List, infinity). +%% Everything here is defined in terms of runmany. +%% The following methods are convient interfaces to runmany. --spec filter(thunk(), [any()], timeout() | infinity) -> [any()]. -filter(Fun, List, Timeout) -> - run_list_fun_in_parallel(filter, Fun, List, Timeout). +%% @doc Same semantics as in module +%% lists. +-spec all/2 :: (el_fun(), list()) -> boolean(). +all(Fun, List) -> + all(Fun, List, 1). -%%============================================================================= -%% Internal API -%%============================================================================= --spec run_list_fun_in_parallel(atom(), thunk(), [any()], timeout() | infinity) -> [any()]. -run_list_fun_in_parallel(ListFun, Fun, List, Timeout) -> - LocalPid = self(), - Pids = - lists:map(fun(E) -> - Pid = - proc_lib:spawn(fun() -> - wait(LocalPid, Fun, - E, Timeout) - end), - {Pid, E} - end, List), - gather(ListFun, Pids). - --spec wait(pid(), thunk(), any(), timeout() | infinity) -> any(). -wait(Parent, Fun, E, Timeout) -> - WaitPid = self(), - Child = spawn(fun() -> - do_f(WaitPid, Fun, E) - end), - - wait(Parent, Child, Timeout). - --spec wait(pid(), pid(), timeout() | infinity) -> any(). -wait(Parent, Child, Timeout) -> - receive - {Child, Ret} -> - Parent ! {self(), Ret} - after Timeout -> - exit(Child, timeout), - Parent ! {self(), timeout} - end. - --spec gather(atom(), [any()]) -> [any()]. -gather(map, PidElementList) -> - map_gather(PidElementList); -gather(ftmap, PidElementList) -> - ftmap_gather(PidElementList); -gather(filter, PidElementList) -> - filter_gather(PidElementList). - --spec map_gather([pid()]) -> [any()]. -map_gather([{Pid, _E} | Rest]) -> - receive - {Pid, {value, Ret}} -> - [Ret|map_gather(Rest)]; - %% timeouts fall here too. Should timeouts be a return value - %% or an exception? I lean toward return value, but the code - %% is easier with the exception. Thoughts? - {Pid, Exception} -> - killall(Rest), - throw(Exception) - end; -map_gather([]) -> - []. - --spec ftmap_gather([pid()]) -> [any()]. -ftmap_gather([{Pid, _E} | Rest]) -> - receive - {Pid, Value} -> [Value|ftmap_gather(Rest)] - end; -ftmap_gather([]) -> - []. - --spec filter_gather([pid()]) -> [any()]. -filter_gather([{Pid, E} | Rest]) -> - receive - {Pid, {value, false}} -> - filter_gather(Rest); - {Pid, {value, true}} -> - [E|filter_gather(Rest)]; - {Pid, {value, NotBool}} -> - killall(Rest), - throw({bad_return_value, NotBool}); - {Pid, Exception} -> - killall(Rest), - throw(Exception) - end; -filter_gather([]) -> - []. - --spec do_f(pid(), thunk(), any()) -> no_return(). -do_f(Parent, F, E) -> +%% @doc Same semantics as in module +%% lists. +-spec all/3 :: (el_fun(), list(), malt()) -> boolean(). +all(Fun, List, Malt) -> try - Result = F(E), - Parent ! {self(), {value, Result}} + runmany(fun (L) -> + B = lists:all(Fun, L), + if + B -> + nil; + true -> + erlang:throw(notall) + end + end, + fun (_A1, _A2) -> + nil + end, + List, Malt), + true catch - _Class:Exception -> - %% Losing class info here, but since throw does not accept - %% that arg anyhow and forces a class of throw it does not - %% matter. - Parent ! {self(), Exception} + throw:notall -> + false end. --spec killall([pid()]) -> ok. -killall([{Pid, _E}|T]) -> - exit(Pid, kill), - killall(T); -killall([]) -> - ok. +%% @doc Same semantics as in module +%% lists. +-spec any/2 :: (fun(), list()) -> boolean(). +any(Fun, List) -> + any(Fun, List, 1). -%%============================================================================= -%% Tests -%%============================================================================= - --ifndef(NOTEST). --include_lib("eunit/include/eunit.hrl"). - -map_good_test() -> - Results = map(fun(_) -> - ok - end, - lists:seq(1, 5), infinity), - ?assertMatch([ok, ok, ok, ok, ok], - Results). - -ftmap_good_test() -> - Results = ftmap(fun(_) -> - ok - end, - lists:seq(1, 3), infinity), - ?assertMatch([{value, ok}, {value, ok}, {value, ok}], - Results). - -filter_good_test() -> - Results = filter(fun(X) -> - X == show - end, - [show, show, remove], infinity), - ?assertMatch([show, show], - Results). - -map_timeout_test() -> - Results = - try - map(fun(T) -> - timer:sleep(T), - T +%% @doc Same semantics as in module +%% lists. +-spec any/3 :: (fun(), list(), malt()) -> boolean(). +any(Fun, List, Malt) -> + try + runmany(fun (L) -> + B = lists:any(Fun, L), + if B -> + erlang:throw(any); + true -> + nil + end end, - [1, 100], 10) - catch - C:E -> {C, E} - end, - ?assertMatch({throw, timeout}, Results). + fun (_A1, _A2) -> + nil + end, + List, Malt) of + _ -> + false + catch throw:any -> + true + end. -ftmap_timeout_test() -> - Results = ftmap(fun(X) -> - timer:sleep(X), - true - end, - [100, 1], 10), - ?assertMatch([timeout, {value, true}], Results). +%% @doc Same semantics as in module +%% lists. +-spec filter/2 :: (fun(), list()) -> list(). +filter(Fun, List) -> + filter(Fun, List, 1). -filter_timeout_test() -> - Results = - try - filter(fun(T) -> - timer:sleep(T), - T == 1 +%% @doc Same semantics as in module +%% lists. +-spec filter/3 :: (fun(), list(), malt()) -> list(). +filter(Fun, List, Malt) -> + runmany(fun (L) -> + lists:filter(Fun, L) + end, + {reverse, fun (A1, A2) -> + A1 ++ A2 + end}, + List, Malt). + +%% Note that with parallel fold there is not foldl and foldr, +%% instead just one fold that can fuse Accumlators. + +%% @doc Like below, but assumes 1 as the Malt. This function is almost useless, +%% and is intended only to aid converting code from using lists to plists. +-spec fold/3 :: (fun(), InitAcc::term(), list()) -> term(). +fold(Fun, InitAcc, List) -> + fold(Fun, Fun, InitAcc, List, 1). + +%% @doc Like below, but uses the Fun as the Fuse by default. +-spec fold/4 :: (fun(), InitAcc::term(), list(), malt()) -> term(). +fold(Fun, InitAcc, List, Malt) -> + fold(Fun, Fun, InitAcc, List, Malt). + +%% @doc fold is more complex when made parallel. There is no foldl and +%% foldr, accumulators aren't passed in any defined order. The list +%% is split into sublists which are folded together. Fun is identical +%% to the function passed to lists:fold[lr], it takes (an element, and +%% the accumulator) and returns -> a new accumulator. It is used for +%% the initial stage of folding sublists. Fuse fuses together the +%% results, it takes (Results1, Result2) and returns -> a new result. +%% By default sublists are fused left to right, each result of a fuse +%% being fed into the first element of the next fuse. The result of +%% the last fuse is the result. +%% +%% Fusing may also run in parallel using a recursive algorithm, +%% by specifying the fuse as {recursive, Fuse}. See +%% the discussion in {@link runmany/4}. +%% +%% Malt is the malt for the initial folding of sublists, and for the +%% possible recursive fuse. +-spec fold/5 :: (fun(), fuse(), InitAcc::term(), list(), malt()) -> term(). +fold(Fun, Fuse, InitAcc, List, Malt) -> + Fun2 = fun (L) -> + lists:foldl(Fun, InitAcc, L) + end, + runmany(Fun2, Fuse, List, Malt). + +%% @doc Similiar to foreach in module +%% lists +%% except it makes no guarantee about the order it processes list elements. +-spec foreach/2 :: (fun(), list()) -> ok. +foreach(Fun, List) -> + foreach(Fun, List, 1). + +%% @doc Similiar to foreach in module +%% lists +%% except it makes no guarantee about the order it processes list elements. +-spec foreach/3 :: (fun(), list(), malt()) -> ok. +foreach(Fun, List, Malt) -> + runmany(fun (L) -> + lists:foreach(Fun, L) + end, + fun (_A1, _A2) -> + ok + end, + List, Malt). + +%% @doc Same semantics as in module +%% lists. +-spec map/2 :: (fun(), list()) -> list(). +map(Fun, List) -> + map(Fun, List, 1). + +%% @doc Same semantics as in module +%% lists. +-spec map/3 :: (fun(), list(), malt()) -> list(). +map(Fun, List, Malt) -> + runmany(fun (L) -> + lists:map(Fun, L) + end, + {reverse, fun (A1, A2) -> + A1 ++ A2 + end}, + List, Malt). + +%% @doc values are returned as {value, term()}. +-spec ftmap/2 :: (fun(), list()) -> list(). +ftmap(Fun, List) -> + map(fun(L) -> + try + {value, Fun(L)} + catch + Class:Type -> + {error, {Class, Type}} + end + end, List). + +%% @doc values are returned as {value, term()}. +-spec ftmap/3 :: (fun(), list(), malt()) -> list(). +ftmap(Fun, List, Malt) -> + map(fun(L) -> + try + {value, Fun(L)} + catch + Class:Type -> + {error, {Class, Type}} + end + end, List, Malt). + +%% @doc Same semantics as in module +%% lists. +-spec partition/2 :: (fun(), list()) -> {list(), list()}. +partition(Fun, List) -> + partition(Fun, List, 1). + +%% @doc Same semantics as in module +%% lists. +-spec partition/3 :: (fun(), list(), malt()) -> {list(), list()}. +partition(Fun, List, Malt) -> + runmany(fun (L) -> + lists:partition(Fun, L) + end, + {reverse, fun ({True1, False1}, {True2, False2}) -> + {True1 ++ True2, False1 ++ False2} + end}, + List, Malt). + +%% SORTMALT needs to be tuned +-define(SORTMALT, 100). + +%% @doc Same semantics as in module +%% lists. +-spec sort/1 :: (list()) -> list(). +sort(List) -> + sort(fun (A, B) -> + A =< B + end, + List). + +%% @doc Same semantics as in module +%% lists. +-spec sort/2 :: (fun(), list()) -> list(). +sort(Fun, List) -> + sort(Fun, List, ?SORTMALT). + +%% @doc This version lets you specify your own malt for sort. +%% +%% sort splits the list into sublists and sorts them, and it merges the +%% sorted lists together. These are done in parallel. Each sublist is +%% sorted in a seperate process, and each merging of results is done in a +%% seperate process. Malt defaults to 100, causing the list to be split into +%% 100-element sublists. +-spec sort/3 :: (fun(), list(), malt()) -> list(). +sort(Fun, List, Malt) -> + Fun2 = fun (L) -> + lists:sort(Fun, L) + end, + Fuse = fun (A1, A2) -> + lists:merge(Fun, A1, A2) + end, + runmany(Fun2, {recursive, Fuse}, List, Malt). + +%% @doc Same semantics as in module +%% lists. +-spec usort/1 :: (list()) -> list(). +usort(List) -> + usort(fun (A, B) -> + A =< B + end, + List). + +%% @doc Same semantics as in module +%% lists. +-spec usort/2 :: (fun(), list()) -> list(). +usort(Fun, List) -> + usort(Fun, List, ?SORTMALT). + +%% @doc This version lets you specify your own malt for usort. +%% +%% usort splits the list into sublists and sorts them, and it merges the +%% sorted lists together. These are done in parallel. Each sublist is +%% sorted in a seperate process, and each merging of results is done in a +%% seperate process. Malt defaults to 100, causing the list to be split into +%% 100-element sublists. +%% +%% usort removes duplicate elments while it sorts. +-spec usort/3 :: (fun(), list(), malt()) -> list(). +usort(Fun, List, Malt) -> + Fun2 = fun (L) -> + lists:usort(Fun, L) + end, + Fuse = fun (A1, A2) -> + lists:umerge(Fun, A1, A2) + end, + runmany(Fun2, {recursive, Fuse}, List, Malt). + +%% @doc Like below, assumes default MapMalt of 1. +-spec mapreduce/2 :: (MapFunc, list()) -> dict() when + MapFunc :: fun((term()) -> DeepListOfKeyValuePairs), + DeepListOfKeyValuePairs :: [DeepListOfKeyValuePairs] | {Key::term(), Value::term()}. +mapreduce(MapFunc, List) -> + mapreduce(MapFunc, List, 1). + +%% Like below, but uses a default reducer that collects all +%% {Key, Value} pairs into a +%% dict, +%% with values {Key, [Value1, Value2...]}. +%% This dict is returned as the result. +mapreduce(MapFunc, List, MapMalt) -> + mapreduce(MapFunc, List, dict:new(), fun add_key/3, MapMalt). + +%% @doc This is a very basic mapreduce. You won't write a +%% Google-rivaling search engine with it. It has no equivalent in +%% lists. Each element in the list is run through the MapFunc, which +%% produces either a {Key, Value} pair, or a lists of key value pairs, +%% or a list of lists of key value pairs...etc. A reducer process runs +%% in parallel with the mapping processes, collecting the key value +%% pairs. It starts with a state given by InitState, and for each +%% {Key, Value} pair that it receives it invokes ReduceFunc(OldState, +%% Key, Value) to compute its new state. mapreduce returns the +%% reducer's final state. +%% +%% MapMalt is the malt for the mapping operation, with a default value of 1, +%% meaning each element of the list is mapped by a seperate process. +%% +%% mapreduce requires OTP R11B, or it may leave monitoring messages in the +%% message queue. +-spec mapreduce/5 :: (MapFunc, list(), InitState::term(), ReduceFunc, malt()) -> dict() when + MapFunc :: fun((term()) -> DeepListOfKeyValuePairs), + DeepListOfKeyValuePairs :: [DeepListOfKeyValuePairs] | {Key::term(), Value::term()}, + ReduceFunc :: fun((OldState::term(), Key::term(), Value::term()) -> NewState::term()). +mapreduce(MapFunc, List, InitState, ReduceFunc, MapMalt) -> + Parent = self(), + {Reducer, ReducerRef} = + erlang:spawn_monitor(fun () -> + reducer(Parent, 0, InitState, ReduceFunc) + end), + MapFunc2 = fun (L) -> + Reducer ! lists:map(MapFunc, L), + 1 + end, + SentMessages = try + runmany(MapFunc2, fun (A, B) -> A+B end, List, MapMalt) + catch + exit:Reason -> + erlang:demonitor(ReducerRef, [flush]), + Reducer ! die, + exit(Reason) end, - [1, 100], 10) - catch - C:E -> {C, E} - end, - ?assertMatch({throw, timeout}, Results). - -map_bad_test() -> - Results = - try - map(fun(_) -> - throw(test_exception) - end, - lists:seq(1, 5), infinity) - catch - C:E -> {C, E} - end, - ?assertMatch({throw, test_exception}, Results). - -ftmap_bad_test() -> - Results = - ftmap(fun(2) -> - throw(test_exception); - (N) -> - N + Reducer ! {mappers, done, SentMessages}, + Results = receive + {Reducer, Results2} -> + Results2; + {'DOWN', _, _, Reducer, Reason2} -> + exit(Reason2) end, - lists:seq(1, 5), infinity), - ?assertMatch([{value, 1}, test_exception, {value, 3}, - {value, 4}, {value, 5}] , Results). + receive + {'DOWN', _, _, Reducer, normal} -> + nil + end, + Results. --endif. +reducer(Parent, NumReceived, State, Func) -> + receive + die -> + nil; + {mappers, done, NumReceived} -> + Parent ! {self (), State}; + Keys -> + reducer(Parent, NumReceived + 1, each_key(State, Func, Keys), Func) + end. + +each_key(State, Func, {Key, Value}) -> + Func(State, Key, Value); +each_key(State, Func, [List|Keys]) -> + each_key(each_key(State, Func, List), Func, Keys); +each_key(State, _, []) -> + State. + +add_key(Dict, Key, Value) -> + case dict:is_key(Key, Dict) of + true -> + dict:append(Key, Value, Dict); + false -> + dict:store(Key, [Value], Dict) + end. + +%% @doc Like below, but assumes a Malt of 1, +%% meaning each element of the list is processed by a seperate process. +-spec runmany/3 :: (fun(), fuse(), list()) -> term(). +runmany(Fun, Fuse, List) -> + runmany(Fun, Fuse, List, 1). + +%% Begin internal stuff (though runmany/4 is exported). + +%% @doc All of the other functions are implemented with runmany. runmany +%% takes a List, splits it into sublists, and starts processes to operate on +%% each sublist, all done according to Malt. Each process passes its sublist +%% into Fun and sends the result back. +%% +%% The results are then fused together to get the final result. There are two +%% ways this can operate, lineraly and recursively. If Fuse is a function, +%% a fuse is done linearly left-to-right on the sublists, the results +%% of processing the first and second sublists being passed to Fuse, then +%% the result of the first fuse and processing the third sublits, and so on. If +%% Fuse is {reverse, FuseFunc}, then a fuse is done right-to-left, the results +%% of processing the second-to-last and last sublists being passed to FuseFunc, +%% then the results of processing the third-to-last sublist and +%% the results of the first fuse, and and so forth. +%% Both methods preserve the original order of the lists elements. +%% +%% To do a recursive fuse, pass Fuse as {recursive, FuseFunc}. +%% The recursive fuse makes no guarantee about the order the results of +%% sublists, or the results of fuses are passed to FuseFunc. It +%% continues fusing pairs of results until it is down to one. +%% +%% Recursive fuse is down in parallel with processing the sublists, and a +%% process is spawned to fuse each pair of results. It is a parallized +%% algorithm. Linear fuse is done after all results of processing sublists +%% have been collected, and can only run in a single process. +%% +%% Even if you pass {recursive, FuseFunc}, a recursive fuse is only done if +%% the malt contains {nodes, NodeList} or {processes, X}. If this is not the +%% case, a linear fuse is done. +-spec runmany/4 :: (fun(([term()]) -> term()), fuse(), list(), malt()) -> term(). +runmany(Fun, Fuse, List, Malt) + when erlang:is_list(Malt) -> + runmany(Fun, Fuse, List, local, no_split, Malt); +runmany(Fun, Fuse, List, Malt) -> + runmany(Fun, Fuse, List, [Malt]). + +runmany(Fun, Fuse, List, Nodes, no_split, [MaltTerm|Malt]) + when erlang:is_integer(MaltTerm) -> + runmany(Fun, Fuse, List, Nodes, MaltTerm, Malt); +runmany(Fun, Fuse, List, local, Split, [{processes, schedulers}|Malt]) -> + %% run a process for each scheduler + S = erlang:system_info(schedulers), + runmany(Fun, Fuse, List, local, Split, [{processes, S}|Malt]); +runmany(Fun, Fuse, List, local, no_split, [{processes, X}|_]=Malt) -> + %% Split the list into X sublists, where X is the number of processes + L = erlang:length(List), + case (L rem X) of + 0 -> + runmany(Fun, Fuse, List, local, (L / X), Malt); + _ -> + runmany(Fun, Fuse, List, local, (L / X) + 1, Malt) + end; +runmany(Fun, Fuse, List, local, Split, [{processes, X}|Malt]) -> + %% run X process on local machine + Nodes = lists:duplicate(X, node()), + runmany(Fun, Fuse, List, Nodes, Split, Malt); +runmany(Fun, Fuse, List, Nodes, Split, [{timeout, X}|Malt]) -> + Parent = erlang:self(), + Timer = proc_lib:spawn(fun () -> + receive + stoptimer -> + Parent ! {timerstopped, erlang:self()} + after X -> + Parent ! {timerrang, erlang:self()}, + receive + stoptimer -> + Parent ! {timerstopped, erlang:self()} + end + end + end), + Ans = try + runmany(Fun, Fuse, List, Nodes, Split, Malt) + catch + %% we really just want the after block, the syntax + %% makes this catch necessary. + willneverhappen -> + nil + after + Timer ! stoptimer, + cleanup_timer(Timer) + end, + Ans; +runmany(Fun, Fuse, List, local, Split, [{nodes, NodeList}|Malt]) -> + Nodes = lists:foldl(fun ({Node, schedulers}, A) -> + X = schedulers_on_node(Node) + 1, + lists:reverse(lists:duplicate(X, Node), A); + ({Node, X}, A) -> + lists:reverse(lists:duplicate(X, Node), A); + (Node, A) -> + [Node|A] + end, + [], NodeList), + runmany(Fun, Fuse, List, Nodes, Split, Malt); +runmany(Fun, {recursive, Fuse}, List, local, Split, []) -> + %% local recursive fuse, for when we weren't invoked with {processes, X} + %% or {nodes, NodeList}. Degenerates recursive fuse into linear fuse. + runmany(Fun, Fuse, List, local, Split, []); +runmany(Fun, Fuse, List, Nodes, no_split, []) -> + %% by default, operate on each element seperately + runmany(Fun, Fuse, List, Nodes, 1, []); +runmany(Fun, Fuse, List, local, Split, []) -> + List2 = splitmany(List, Split), + local_runmany(Fun, Fuse, List2); +runmany(Fun, Fuse, List, Nodes, Split, []) -> + List2 = splitmany(List, Split), + cluster_runmany(Fun, Fuse, List2, Nodes). + +cleanup_timer(Timer) -> + receive + {timerrang, Timer} -> + cleanup_timer(Timer); + {timerstopped, Timer} -> + nil + end. + +schedulers_on_node(Node) -> + case erlang:get(ec_plists_schedulers_on_nodes) of + undefined -> + X = determine_schedulers(Node), + erlang:put(ec_plists_schedulers_on_nodes, + dict:store(Node, X, dict:new())), + X; + Dict -> + case dict:is_key(Node, Dict) of + true -> + dict:fetch(Node, Dict); + false -> + X = determine_schedulers(Node), + erlang:put(ec_plists_schedulers_on_nodes, + dict:store(Node, X, Dict)), + X + end + end. + +determine_schedulers(Node) -> + Parent = erlang:self(), + Child = proc_lib:spawn(Node, fun () -> + Parent ! {self(), erlang:system_info(schedulers)} + end), + erlang:monitor(process, Child), + receive + {Child, X} -> + receive + {'DOWN', _, _, Child, _Reason} -> + nil + end, + X; + {'DOWN', _, _, Child, Reason} when Reason =/= normal -> + 0 + end. + +%% @doc local runmany, for when we weren't invoked with {processes, X} +%% or {nodes, NodeList}. Every sublist is processed in parallel. +local_runmany(Fun, Fuse, List) -> + Parent = self (), + Pids = lists:map(fun (L) -> + F = fun () -> + Parent ! {self (), Fun(L)} + end, + {Pid, _} = erlang:spawn_monitor(F), + Pid + end, + List), + Answers = try + lists:map(fun receivefrom/1, Pids) + catch + throw:Message -> + {BadPid, Reason} = Message, + handle_error(BadPid, Reason, Pids) + end, + lists:foreach(fun (Pid) -> + normal_cleanup(Pid) + end, Pids), + fuse(Fuse, Answers). + +receivefrom(Pid) -> + receive + {Pid, R} -> + R; + {'DOWN', _, _, BadPid, Reason} when Reason =/= normal -> + erlang:throw({BadPid, Reason}); + {timerrang, _} -> + erlang:throw({nil, timeout}) + end. + +%% Convert List into [{Number, Sublist}] +cluster_runmany(Fun, Fuse, List, Nodes) -> + {List2, _} = lists:foldl(fun (X, {L, Count}) -> + {[{Count, X}|L], Count+1} + end, + {[], 0}, List), + cluster_runmany(Fun, Fuse, List2, Nodes, [], []). + +%% @doc Add a pair of results into the TaskList as a fusing task +cluster_runmany(Fun, {recursive, Fuse}, [], Nodes, Running, + [{_, R1}, {_, R2}|Results]) -> + cluster_runmany(Fun, {recursive, Fuse}, [{fuse, R1, R2}], Nodes, + Running, Results); +cluster_runmany(_, {recursive, _Fuse}, [], _Nodes, [], [{_, Result}]) -> + %% recursive fuse done, return result + Result; +cluster_runmany(_, {recursive, _Fuse}, [], _Nodes, [], []) -> + %% edge case where we are asked to do nothing + []; +cluster_runmany(_, Fuse, [], _Nodes, [], Results) -> + %% We're done, now we just have to [linear] fuse the results + fuse(Fuse, lists:map(fun ({_, R}) -> + R + end, + lists:sort(fun ({A, _}, {B, _}) -> + A =< B + end, + lists:reverse(Results)))); +cluster_runmany(Fun, Fuse, [Task|TaskList], [N|Nodes], Running, Results) -> +%% We have a ready node and a sublist or fuse to be processed, so we start +%% a new process + + Parent = erlang:self(), + case Task of + {Num, L2} -> + Fun2 = fun () -> + Parent ! {erlang:self(), Num, Fun(L2)} + end; + {fuse, R1, R2} -> + {recursive, FuseFunc} = Fuse, + Fun2 = fun () -> + Parent ! {erlang:self(), fuse, FuseFunc(R1, R2)} + end + end, + Fun3 = fun () -> + try + Fun2() + catch + exit:siblingdied -> + ok; + exit:Reason -> + Parent ! {erlang:self(), error, Reason}; + error:R -> + Parent ! {erlang:self(), error, {R, erlang:get_stacktrace()}}; + throw:R -> + Parent ! {erlang:self(), error, {{nocatch, R}, erlang:get_stacktrace()}} + end + end, + Pid = proc_lib:spawn(N, Fun3), + erlang:monitor(process, Pid), + cluster_runmany(Fun, Fuse, TaskList, Nodes, [{Pid, N, Task}|Running], Results); +cluster_runmany(Fun, Fuse, TaskList, Nodes, Running, Results) when length(Running) > 0 -> + %% We can't start a new process, but can watch over already running ones + receive + {_Pid, error, Reason} -> + RunningPids = lists:map(fun ({Pid, _, _}) -> + Pid + end, + Running), + handle_error(junkvalue, Reason, RunningPids); + {Pid, Num, Result} -> + %% throw out the exit message, Reason should be + %% normal, noproc, or noconnection + receive + {'DOWN', _, _, Pid, _Reason} -> + nil + end, + {Running2, FinishedNode, _} = delete_running(Pid, Running, []), + cluster_runmany(Fun, Fuse, TaskList, + [FinishedNode|Nodes], Running2, [{Num, Result}|Results]); + {timerrang, _} -> + RunningPids = lists:map(fun ({Pid, _, _}) -> + Pid + end, + Running), + handle_error(nil, timeout, RunningPids); + %% node failure + {'DOWN', _, _, Pid, noconnection} -> + {Running2, _DeadNode, Task} = delete_running(Pid, Running, []), + cluster_runmany(Fun, Fuse, [Task|TaskList], Nodes, + Running2, Results); + %% could a noproc exit message come before the message from + %% the process? we are assuming it can't. + %% this clause is unlikely to get invoked due to cluster_runmany's + %% spawned processes. It will still catch errors in mapreduce's + %% reduce process, however. + {'DOWN', _, _, BadPid, Reason} when Reason =/= normal -> + RunningPids = lists:map(fun ({Pid, _, _}) -> + Pid + end, + Running), + handle_error(BadPid, Reason, RunningPids) + end; +cluster_runmany(_, _, [_Non|_Empty], []=_Nodes, []=_Running, _) -> +%% We have data, but no nodes either available or occupied + erlang:exit(allnodescrashed). + +delete_running(Pid, [{Pid, Node, List}|Running], Acc) -> + {Running ++ Acc, Node, List}; +delete_running(Pid, [R|Running], Acc) -> + delete_running(Pid, Running, [R|Acc]). + +handle_error(BadPid, Reason, Pids) -> + lists:foreach(fun (Pid) -> + erlang:exit(Pid, siblingdied) + end, Pids), + lists:foreach(fun (Pid) -> + error_cleanup(Pid, BadPid) + end, Pids), + erlang:exit(Reason). + +error_cleanup(BadPid, BadPid) -> + ok; +error_cleanup(Pid, BadPid) -> + receive + {Pid, _} -> + error_cleanup(Pid, BadPid); + {Pid, _, _} -> + error_cleanup(Pid, BadPid); + {'DOWN', _, _, Pid, _Reason} -> + ok + end. + +normal_cleanup(Pid) -> + receive + {'DOWN', _, _, Pid, _Reason} -> + ok + end. + +%% edge case +fuse(_, []) -> + []; +fuse({reverse, _}=Fuse, Results) -> + [RL|ResultsR] = lists:reverse(Results), + fuse(Fuse, ResultsR, RL); +fuse(Fuse, [R1|Results]) -> + fuse(Fuse, Results, R1). + +fuse({reverse, FuseFunc}=Fuse, [R2|Results], R1) -> + fuse(Fuse, Results, FuseFunc(R2, R1)); +fuse(Fuse, [R2|Results], R1) -> + fuse(Fuse, Results, Fuse(R1, R2)); +fuse(_, [], R) -> + R. + +%% @doc Splits a list into a list of sublists, each of size Size, +%% except for the last element which is less if the original list +%% could not be evenly divided into Size-sized lists. +splitmany(List, Size) -> + splitmany(List, [], Size). + +splitmany([], Acc, _) -> + lists:reverse(Acc); +splitmany(List, Acc, Size) -> + {Top, NList} = split(Size, List), + splitmany(NList, [Top|Acc], Size). + +%% @doc Like lists:split, except it splits a list smaller than its first +%% parameter +split(Size, List) -> + split(Size, List, []). + +split(0, List, Acc) -> + {lists:reverse(Acc), List}; +split(Size, [H|List], Acc) -> + split(Size - 1, List, [H|Acc]); +split(_, [], Acc) -> + {lists:reverse(Acc), []}. diff --git a/src/ec_semver.erl b/src/ec_semver.erl index 4dc83e9..ac810e5 100644 --- a/src/ec_semver.erl +++ b/src/ec_semver.erl @@ -1,3 +1,4 @@ + %%%------------------------------------------------------------------- %%% @copyright (C) 2011, Erlware LLC %%% @doc @@ -8,6 +9,7 @@ -module(ec_semver). -export([parse/1, + format/1, eql/2, gt/2, gte/2, @@ -27,15 +29,20 @@ %%% Public Types %%%=================================================================== --type major_minor_patch() :: - non_neg_integer() - | {non_neg_integer(), non_neg_integer()} - | {non_neg_integer(), non_neg_integer(), non_neg_integer()}. +-type version_element() :: non_neg_integer() | binary(). --type alpha_part() :: integer() | binary(). +-type major_minor_patch_minpatch() :: + version_element() + | {version_element(), version_element()} + | {version_element(), version_element(), version_element()} + | {version_element(), version_element(), + version_element(), version_element()}. --type semver() :: {major_minor_patch(), {PreReleaseVersion::[alpha_part()], - BuildVersion::[alpha_part()]}}. +-type alpha_part() :: integer() | binary() | string(). +-type alpha_info() :: {PreRelease::[alpha_part()], + BuildVersion::[alpha_part()]}. + +-type semver() :: {major_minor_patch_minpatch(), alpha_info()}. -type version_string() :: string() | binary(). @@ -48,12 +55,58 @@ %% @doc parse a string or binary into a valid semver representation -spec parse(any_version()) -> semver(). parse(Version) when erlang:is_list(Version) -> - ec_semver_parser:parse(Version); + case ec_semver_parser:parse(Version) of + {fail, _} -> + {erlang:iolist_to_binary(Version), {[],[]}}; + Good -> + Good + end; parse(Version) when erlang:is_binary(Version) -> - ec_semver_parser:parse(Version); + case ec_semver_parser:parse(Version) of + {fail, _} -> + {Version, {[],[]}}; + Good -> + Good + end; parse(Version) -> Version. +-spec format(semver()) -> iolist(). +format({Maj, {AlphaPart, BuildPart}}) + when erlang:is_integer(Maj); + erlang:is_binary(Maj) -> + [format_version_part(Maj), + format_vsn_rest(<<"-">>, AlphaPart), + format_vsn_rest(<<"+">>, BuildPart)]; +format({{Maj, Min}, {AlphaPart, BuildPart}}) -> + [format_version_part(Maj), ".", + format_version_part(Min), + format_vsn_rest(<<"-">>, AlphaPart), + format_vsn_rest(<<"+">>, BuildPart)]; +format({{Maj, Min, Patch}, {AlphaPart, BuildPart}}) -> + [format_version_part(Maj), ".", + format_version_part(Min), ".", + format_version_part(Patch), + format_vsn_rest(<<"-">>, AlphaPart), + format_vsn_rest(<<"+">>, BuildPart)]; +format({{Maj, Min, Patch, MinPatch}, {AlphaPart, BuildPart}}) -> + [format_version_part(Maj), ".", + format_version_part(Min), ".", + format_version_part(Patch), ".", + format_version_part(MinPatch), + format_vsn_rest(<<"-">>, AlphaPart), + format_vsn_rest(<<"+">>, BuildPart)]. + +-spec format_version_part(integer() | binary()) -> iolist(). +format_version_part(Vsn) + when erlang:is_integer(Vsn) -> + erlang:integer_to_list(Vsn); +format_version_part(Vsn) + when erlang:is_binary(Vsn) -> + Vsn. + + + %% @doc test for quality between semver versions -spec eql(any_version(), any_version()) -> boolean(). eql(VsnA, VsnB) -> @@ -141,8 +194,8 @@ between(Vsn1, Vsn2, VsnMatch) -> %% revisions and "~> 2.6.5" is pessimistic about future minor %% revisions. %% -%% "~> 2.6" matches cookbooks >= 2.6.0 AND < 3.0.0 -%% "~> 2.6.5" matches cookbooks >= 2.6.5 AND < 2.7.0 +%% "~> 2.6" matches cookbooks >= 2.6.0 AND < 3.0.0 +%% "~> 2.6.5" matches cookbooks >= 2.6.5 AND < 2.7.0 pes(VsnA, VsnB) -> internal_pes(parse(VsnA), parse(VsnB)). @@ -152,17 +205,20 @@ pes(VsnA, VsnB) -> %% @doc helper function for the peg grammer to parse the iolist into a semver -spec internal_parse_version(iolist()) -> semver(). internal_parse_version([MMP, AlphaPart, BuildPart, _]) -> - {parse_major_minor_patch(MMP), {parse_alpha_part(AlphaPart), - parse_alpha_part(BuildPart)}}. + {parse_major_minor_patch_minpatch(MMP), {parse_alpha_part(AlphaPart), + parse_alpha_part(BuildPart)}}. %% @doc helper function for the peg grammer to parse the iolist into a major_minor_patch --spec parse_major_minor_patch(iolist()) -> major_minor_patch(). -parse_major_minor_patch([MajVsn, [], []]) -> +-spec parse_major_minor_patch_minpatch(iolist()) -> major_minor_patch_minpatch(). +parse_major_minor_patch_minpatch([MajVsn, [], [], []]) -> MajVsn; -parse_major_minor_patch([MajVsn, [<<".">>, MinVsn], []]) -> +parse_major_minor_patch_minpatch([MajVsn, [<<".">>, MinVsn], [], []]) -> {MajVsn, MinVsn}; -parse_major_minor_patch([MajVsn, [<<".">>, MinVsn], [<<".">>, PatchVsn]]) -> - {MajVsn, MinVsn, PatchVsn}. +parse_major_minor_patch_minpatch([MajVsn, [<<".">>, MinVsn], [<<".">>, PatchVsn], []]) -> + {MajVsn, MinVsn, PatchVsn}; +parse_major_minor_patch_minpatch([MajVsn, [<<".">>, MinVsn], + [<<".">>, PatchVsn], [<<".">>, MinPatch]]) -> + {MajVsn, MinVsn, PatchVsn, MinPatch}. %% @doc helper function for the peg grammer to parse the iolist into an alpha part -spec parse_alpha_part(iolist()) -> [alpha_part()]. @@ -189,32 +245,59 @@ format_alpha_part([<<".">>, AlphaPart]) -> %%%=================================================================== %%% Internal Functions %%%=================================================================== +-spec to_list(integer() | binary() | string()) -> string() | binary(). +to_list(Detail) when erlang:is_integer(Detail) -> + erlang:integer_to_list(Detail); +to_list(Detail) when erlang:is_list(Detail); erlang:is_binary(Detail) -> + Detail. + +-spec format_vsn_rest(binary() | string(), [integer() | binary()]) -> iolist(). +format_vsn_rest(_TypeMark, []) -> + []; +format_vsn_rest(TypeMark, [Head | Rest]) -> + [TypeMark, Head | + [[".", to_list(Detail)] || Detail <- Rest]]. + %% @doc normalize the semver so they can be compared -spec normalize(semver()) -> semver(). normalize({Vsn, Rest}) - when erlang:is_integer(Vsn) -> - {{Vsn, 0, 0}, Rest}; + when erlang:is_binary(Vsn); + erlang:is_integer(Vsn) -> + {{Vsn, 0, 0, 0}, Rest}; normalize({{Maj, Min}, Rest}) -> - {{Maj, Min, 0}, Rest}; -normalize(Other) -> + {{Maj, Min, 0, 0}, Rest}; +normalize({{Maj, Min, Patch}, Rest}) -> + {{Maj, Min, Patch, 0}, Rest}; +normalize(Other = {{_, _, _, _}, {_,_}}) -> Other. %% @doc to do the pessimistic compare we need a parsed semver. This is %% the internal implementation of the of the pessimistic run. The %% external just ensures that versions are parsed. -internal_pes(VsnA, {{LM, LMI}, _}) -> +-spec internal_pes(semver(), semver()) -> boolean(). +internal_pes(VsnA, {{LM, LMI}, _}) + when erlang:is_integer(LM), + erlang:is_integer(LMI) -> gte(VsnA, {{LM, LMI, 0}, {[], []}}) andalso - lt(VsnA, {{LM + 1, 0, 0}, {[], []}}); -internal_pes(VsnA, {{LM, LMI, LP}, _}) -> + lt(VsnA, {{LM + 1, 0, 0, 0}, {[], []}}); +internal_pes(VsnA, {{LM, LMI, LP}, _}) + when erlang:is_integer(LM), + erlang:is_integer(LMI), + erlang:is_integer(LP) -> gte(VsnA, {{LM, LMI, LP}, {[], []}}) andalso - lt(VsnA, {{LM, LMI + 1, 0}, {[], []}}); + lt(VsnA, {{LM, LMI + 1, 0, 0}, {[], []}}); +internal_pes(VsnA, {{LM, LMI, LP, LMP}, _}) + when erlang:is_integer(LM), + erlang:is_integer(LMI), + erlang:is_integer(LP), + erlang:is_integer(LMP) -> + gte(VsnA, {{LM, LMI, LP, LMP}, {[], []}}) + andalso + lt(VsnA, {{LM, LMI, LP + 1, 0}, {[], []}}); internal_pes(Vsn, LVsn) -> gte(Vsn, LVsn). - - - %%%=================================================================== %%% Test Functions %%%=================================================================== @@ -231,25 +314,43 @@ eql_test() -> "1.0.0")), ?assertMatch(true, eql("1.0.0", "1")), + ?assertMatch(true, eql("1.0.0.0", + "1")), ?assertMatch(true, eql("1.0+alpha.1", "1.0.0+alpha.1")), ?assertMatch(true, eql("1.0-alpha.1+build.1", "1.0.0-alpha.1+build.1")), + ?assertMatch(true, eql("1.0-alpha.1+build.1", + "1.0.0.0-alpha.1+build.1")), + ?assertMatch(true, eql("aa", "aa")), + ?assertMatch(true, eql("AA.BB", "AA.BB")), + ?assertMatch(true, eql("BBB-super", "BBB-super")), ?assertMatch(true, not eql("1.0.0", "1.0.1")), ?assertMatch(true, not eql("1.0.0-alpha", "1.0.1+alpha")), ?assertMatch(true, not eql("1.0.0+build.1", - "1.0.1+build.2")). - + "1.0.1+build.2")), + ?assertMatch(true, not eql("1.0.0.0+build.1", + "1.0.0.1+build.2")), + ?assertMatch(true, not eql("FFF", "BBB")), + ?assertMatch(true, not eql("1", "1BBBB")). gt_test() -> ?assertMatch(true, gt("1.0.0-alpha.1", "1.0.0-alpha")), + ?assertMatch(true, gt("1.0.0.1-alpha.1", + "1.0.0.1-alpha")), + ?assertMatch(true, gt("1.0.0.4-alpha.1", + "1.0.0.2-alpha")), + ?assertMatch(true, gt("1.0.0.0-alpha.1", + "1.0.0-alpha")), ?assertMatch(true, gt("1.0.0-beta.2", "1.0.0-alpha.1")), ?assertMatch(true, gt("1.0.0-beta.11", "1.0.0-beta.2")), + ?assertMatch(true, gt("1.0.0-beta.11", + "1.0.0.0-beta.2")), ?assertMatch(true, gt("1.0.0-rc.1", "1.0.0-beta.11")), ?assertMatch(true, gt("1.0.0-rc.1+build.1", "1.0.0-rc.1")), ?assertMatch(true, gt("1.0.0", "1.0.0-rc.1+build.1")), @@ -257,10 +358,16 @@ gt_test() -> ?assertMatch(true, gt("1.3.7+build", "1.0.0+0.3.7")), ?assertMatch(true, gt("1.3.7+build.2.b8f12d7", "1.3.7+build")), + ?assertMatch(true, gt("1.3.7+build.2.b8f12d7", + "1.3.7.0+build")), ?assertMatch(true, gt("1.3.7+build.11.e0f985a", "1.3.7+build.2.b8f12d7")), + ?assertMatch(true, gt("aa.cc", + "aa.bb")), ?assertMatch(true, not gt("1.0.0-alpha", "1.0.0-alpha.1")), + ?assertMatch(true, not gt("1.0.0-alpha", + "1.0.0.0-alpha.1")), ?assertMatch(true, not gt("1.0.0-alpha.1", "1.0.0-beta.2")), ?assertMatch(true, not gt("1.0.0-beta.2", @@ -283,6 +390,10 @@ gt_test() -> "1.0.0-alpha")), ?assertMatch(true, not gt("1", "1.0.0")), + ?assertMatch(true, not gt("aa.bb", + "aa.bb")), + ?assertMatch(true, not gt("aa.cc", + "aa.dd")), ?assertMatch(true, not gt("1.0", "1.0.0")), ?assertMatch(true, not gt("1.0.0", @@ -295,12 +406,16 @@ gt_test() -> lt_test() -> ?assertMatch(true, lt("1.0.0-alpha", "1.0.0-alpha.1")), + ?assertMatch(true, lt("1.0.0-alpha", + "1.0.0.0-alpha.1")), ?assertMatch(true, lt("1.0.0-alpha.1", "1.0.0-beta.2")), ?assertMatch(true, lt("1.0.0-beta.2", "1.0.0-beta.11")), ?assertMatch(true, lt("1.0.0-beta.11", "1.0.0-rc.1")), + ?assertMatch(true, lt("1.0.0.1-beta.11", + "1.0.0.1-rc.1")), ?assertMatch(true, lt("1.0.0-rc.1", "1.0.0-rc.1+build.1")), ?assertMatch(true, lt("1.0.0-rc.1+build.1", @@ -317,12 +432,17 @@ lt_test() -> "1.0.0-alpha")), ?assertMatch(true, not lt("1", "1.0.0")), + ?assertMatch(true, lt("1", + "1.0.0.1")), + ?assertMatch(true, lt("AA.DD", + "AA.EE")), ?assertMatch(true, not lt("1.0", "1.0.0")), - ?assertMatch(true, not lt("1.0.0", + ?assertMatch(true, not lt("1.0.0.0", "1")), ?assertMatch(true, not lt("1.0+alpha.1", "1.0.0+alpha.1")), + ?assertMatch(true, not lt("AA.DD", "AA.CC")), ?assertMatch(true, not lt("1.0-alpha.1+build.1", "1.0.0-alpha.1+build.1")), ?assertMatch(true, not lt("1.0.0-alpha.1", @@ -341,7 +461,6 @@ lt_test() -> ?assertMatch(true, not lt("1.3.7+build.11.e0f985a", "1.3.7+build.2.b8f12d7")). - gte_test() -> ?assertMatch(true, gte("1.0.0-alpha", "1.0.0-alpha")), @@ -355,18 +474,25 @@ gte_test() -> ?assertMatch(true, gte("1.0.0", "1")), + ?assertMatch(true, gte("1.0.0.0", + "1")), + ?assertMatch(true, gte("1.0+alpha.1", "1.0.0+alpha.1")), ?assertMatch(true, gte("1.0-alpha.1+build.1", "1.0.0-alpha.1+build.1")), + ?assertMatch(true, gte("1.0.0-alpha.1+build.1", + "1.0.0.0-alpha.1+build.1")), ?assertMatch(true, gte("1.0.0-alpha.1", "1.0.0-alpha")), ?assertMatch(true, gte("1.0.0-beta.2", "1.0.0-alpha.1")), ?assertMatch(true, gte("1.0.0-beta.11", "1.0.0-beta.2")), + ?assertMatch(true, gte("aa.bb", "aa.bb")), + ?assertMatch(true, gte("dd", "aa")), ?assertMatch(true, gte("1.0.0-rc.1", "1.0.0-beta.11")), ?assertMatch(true, gte("1.0.0-rc.1+build.1", "1.0.0-rc.1")), ?assertMatch(true, gte("1.0.0", "1.0.0-rc.1+build.1")), @@ -378,6 +504,7 @@ gte_test() -> "1.3.7+build.2.b8f12d7")), ?assertMatch(true, not gte("1.0.0-alpha", "1.0.0-alpha.1")), + ?assertMatch(true, not gte("CC", "DD")), ?assertMatch(true, not gte("1.0.0-alpha.1", "1.0.0-beta.2")), ?assertMatch(true, not gte("1.0.0-beta.2", @@ -429,25 +556,29 @@ lte_test() -> "1")), ?assertMatch(true, lte("1.0+alpha.1", "1.0.0+alpha.1")), + ?assertMatch(true, lte("1.0.0.0+alpha.1", + "1.0.0+alpha.1")), ?assertMatch(true, lte("1.0-alpha.1+build.1", "1.0.0-alpha.1+build.1")), - ?assertMatch(true, not lt("1.0.0-alpha.1", + ?assertMatch(true, lte("aa","cc")), + ?assertMatch(true, lte("cc","cc")), + ?assertMatch(true, not lte("1.0.0-alpha.1", "1.0.0-alpha")), - ?assertMatch(true, not lt("1.0.0-beta.2", + ?assertMatch(true, not lte("cc", "aa")), + ?assertMatch(true, not lte("1.0.0-beta.2", "1.0.0-alpha.1")), - ?assertMatch(true, not lt("1.0.0-beta.11", + ?assertMatch(true, not lte("1.0.0-beta.11", "1.0.0-beta.2")), - ?assertMatch(true, not lt("1.0.0-rc.1", "1.0.0-beta.11")), - ?assertMatch(true, not lt("1.0.0-rc.1+build.1", "1.0.0-rc.1")), - ?assertMatch(true, not lt("1.0.0", "1.0.0-rc.1+build.1")), - ?assertMatch(true, not lt("1.0.0+0.3.7", "1.0.0")), - ?assertMatch(true, not lt("1.3.7+build", "1.0.0+0.3.7")), - ?assertMatch(true, not lt("1.3.7+build.2.b8f12d7", + ?assertMatch(true, not lte("1.0.0-rc.1", "1.0.0-beta.11")), + ?assertMatch(true, not lte("1.0.0-rc.1+build.1", "1.0.0-rc.1")), + ?assertMatch(true, not lte("1.0.0", "1.0.0-rc.1+build.1")), + ?assertMatch(true, not lte("1.0.0+0.3.7", "1.0.0")), + ?assertMatch(true, not lte("1.3.7+build", "1.0.0+0.3.7")), + ?assertMatch(true, not lte("1.3.7+build.2.b8f12d7", "1.3.7+build")), - ?assertMatch(true, not lt("1.3.7+build.11.e0f985a", + ?assertMatch(true, not lte("1.3.7+build.11.e0f985a", "1.3.7+build.2.b8f12d7")). - between_test() -> ?assertMatch(true, between("1.0.0-alpha", "1.0.0-alpha.3", @@ -464,6 +595,10 @@ between_test() -> ?assertMatch(true, between("1.0.0-rc.1", "1.0.0-rc.1+build.3", "1.0.0-rc.1+build.1")), + + ?assertMatch(true, between("1.0.0.0-rc.1", + "1.0.0-rc.1+build.3", + "1.0.0-rc.1+build.1")), ?assertMatch(true, between("1.0.0-rc.1+build.1", "1.0.0", "1.0.0-rc.33")), @@ -488,6 +623,10 @@ between_test() -> ?assertMatch(true, between("1.0", "1.0.0", "1.0.0")), + + ?assertMatch(true, between("1.0", + "1.0.0.0", + "1.0.0.0")), ?assertMatch(true, between("1.0.0", "1", "1")), @@ -497,6 +636,9 @@ between_test() -> ?assertMatch(true, between("1.0-alpha.1+build.1", "1.0.0-alpha.1+build.1", "1.0.0-alpha.1+build.1")), + ?assertMatch(true, between("aaa", + "ddd", + "cc")), ?assertMatch(true, not between("1.0.0-alpha.1", "1.0.0-alpha.22", "1.0.0")), @@ -506,13 +648,16 @@ between_test() -> ?assertMatch(true, not between("1.0.0-beta.1", "1.0.0-beta.11", "1.0.0-alpha")), - ?assertMatch(true, not between("1.0.0-beta.11", "1.0.0-rc.1", "1.0.0-rc.22")). + ?assertMatch(true, not between("1.0.0-beta.11", "1.0.0-rc.1", + "1.0.0-rc.22")), + ?assertMatch(true, not between("aaa", "ddd", "zzz")). pes_test() -> ?assertMatch(true, pes("2.6.0", "2.6")), ?assertMatch(true, pes("2.7", "2.6")), ?assertMatch(true, pes("2.8", "2.6")), ?assertMatch(true, pes("2.9", "2.6")), + ?assertMatch(true, pes("A.B", "A.A")), ?assertMatch(true, not pes("3.0.0", "2.6")), ?assertMatch(true, not pes("2.5", "2.6")), ?assertMatch(true, pes("2.6.5", "2.6.5")), @@ -520,7 +665,29 @@ pes_test() -> ?assertMatch(true, pes("2.6.7", "2.6.5")), ?assertMatch(true, pes("2.6.8", "2.6.5")), ?assertMatch(true, pes("2.6.9", "2.6.5")), + ?assertMatch(true, pes("2.6.0.9", "2.6.0.5")), ?assertMatch(true, not pes("2.7", "2.6.5")), + ?assertMatch(true, not pes("2.1.7", "2.1.6.5")), + ?assertMatch(true, not pes("A.A", "A.B")), ?assertMatch(true, not pes("2.5", "2.6.5")). +version_format_test() -> + ?assertEqual(["1", [], []], format({1, {[],[]}})), + ?assertEqual(["1", ".", "2", ".", "34", [], []], format({{1,2,34},{[],[]}})), + ?assertEqual(<<"a">>, erlang:iolist_to_binary(format({<<"a">>, {[],[]}}))), + ?assertEqual(<<"a.b">>, erlang:iolist_to_binary(format({{<<"a">>,<<"b">>}, {[],[]}}))), + ?assertEqual(<<"1">>, erlang:iolist_to_binary(format({1, {[],[]}}))), + ?assertEqual(<<"1.2">>, erlang:iolist_to_binary(format({{1,2}, {[],[]}}))), + ?assertEqual(<<"1.2.2">>, erlang:iolist_to_binary(format({{1,2,2}, {[],[]}}))), + ?assertEqual(<<"1.99.2">>, erlang:iolist_to_binary(format({{1,99,2}, {[],[]}}))), + ?assertEqual(<<"1.99.2-alpha">>, erlang:iolist_to_binary(format({{1,99,2}, {[<<"alpha">>],[]}}))), + ?assertEqual(<<"1.99.2-alpha.1">>, erlang:iolist_to_binary(format({{1,99,2}, {[<<"alpha">>,1], []}}))), + ?assertEqual(<<"1.99.2+build.1.a36">>, + erlang:iolist_to_binary(format({{1,99,2}, {[], [<<"build">>, 1, <<"a36">>]}}))), + ?assertEqual(<<"1.99.2.44+build.1.a36">>, + erlang:iolist_to_binary(format({{1,99,2,44}, {[], [<<"build">>, 1, <<"a36">>]}}))), + ?assertEqual(<<"1.99.2-alpha.1+build.1.a36">>, + erlang:iolist_to_binary(format({{1,99,2}, {[<<"alpha">>, 1], [<<"build">>, 1, <<"a36">>]}}))), + ?assertEqual(<<"1">>, erlang:iolist_to_binary(format({1, {[],[]}}))). + -endif. diff --git a/src/ec_semver_parser.peg b/src/ec_semver_parser.peg index 9636d95..09779ae 100644 --- a/src/ec_semver_parser.peg +++ b/src/ec_semver_parser.peg @@ -1,11 +1,12 @@ -semver <- major_minor_patch ("-" alpha_part ("." alpha_part)*)? ("+" alpha_part ("." alpha_part)*)? !. +semver <- major_minor_patch_min_patch ("-" alpha_part ("." alpha_part)*)? ("+" alpha_part ("." alpha_part)*)? !. ` ec_semver:internal_parse_version(Node) ` ; -major_minor_patch <- version_part ("." version_part)? ("." version_part)? ; +major_minor_patch_min_patch <- version_part ("." version_part)? ("." version_part)? ("." version_part)? ; -version_part <- [0-9]+ `erlang:list_to_integer(erlang:binary_to_list(erlang:iolist_to_binary(Node)))` ; +version_part <- numeric_part / alpha_part ; -alpha_part <- [A-Za-z0-9-]+ ; +numeric_part <- [0-9]+ `erlang:list_to_integer(erlang:binary_to_list(erlang:iolist_to_binary(Node)))` ; +alpha_part <- [A-Za-z0-9]+ `erlang:iolist_to_binary(Node)` ; %% This only exists to get around a bug in erlang where if %% warnings_as_errors is specified `nowarn` directives are ignored diff --git a/src/erlware_commons.app.src b/src/erlware_commons.app.src index 8aaa5f3..042c56a 100644 --- a/src/erlware_commons.app.src +++ b/src/erlware_commons.app.src @@ -1,7 +1,7 @@ %% -*- mode: Erlang; fill-column: 75; comment-column: 50; -*- {application, erlware_commons, [{description, "Additional standard library for Erlang"}, - {vsn, "0.8.0"}, + {vsn, "semver"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib]}]}. diff --git a/test/ec_plists_tests.erl b/test/ec_plists_tests.erl new file mode 100644 index 0000000..7acefe6 --- /dev/null +++ b/test/ec_plists_tests.erl @@ -0,0 +1,75 @@ +%%% @copyright Erlware, LLC. +-module(ec_plists_tests). + +-include_lib("eunit/include/eunit.hrl"). + +%%%=================================================================== +%%% Tests +%%%=================================================================== + +map_good_test() -> + Results = ec_plists:map(fun(_) -> + ok + end, + lists:seq(1, 5)), + ?assertMatch([ok, ok, ok, ok, ok], + Results). + +ftmap_good_test() -> + Results = ec_plists:ftmap(fun(_) -> + ok + end, + lists:seq(1, 3)), + ?assertMatch([{value, ok}, {value, ok}, {value, ok}], + Results). + +filter_good_test() -> + Results = ec_plists:filter(fun(X) -> + X == show + end, + [show, show, remove]), + ?assertMatch([show, show], + Results). + +map_timeout_test() -> + ?assertExit(timeout, + ec_plists:map(fun(T) -> + timer:sleep(T), + T + end, + [1, 100], {timeout, 10})). + +ftmap_timeout_test() -> + ?assertExit(timeout, + ec_plists:ftmap(fun(X) -> + timer:sleep(X), + true + end, + [100, 1], {timeout, 10})). + +filter_timeout_test() -> + ?assertExit(timeout, + ec_plists:filter(fun(T) -> + timer:sleep(T), + T == 1 + end, + [1, 100], {timeout, 10})). + +map_bad_test() -> + ?assertExit({{nocatch,test_exception}, _}, + ec_plists:map(fun(_) -> + erlang:throw(test_exception) + end, + lists:seq(1, 5))). + + +ftmap_bad_test() -> + Results = + ec_plists:ftmap(fun(2) -> + erlang:throw(test_exception); + (N) -> + N + end, + lists:seq(1, 5)), + ?assertMatch([{value, 1}, {error,{throw,test_exception}}, {value, 3}, + {value, 4}, {value, 5}] , Results).