From c82bf703ae43ebc0c69a474a001d80ae74bb9b94 Mon Sep 17 00:00:00 2001 From: Tomas Abrahamsson Date: Fri, 2 Jan 2015 01:07:01 +0100 Subject: [PATCH 1/3] Replace inttest external dependency to gpb with a mock In inttest/proto_gpb, replace the external dependency to gpb, a protocol buffer compiler, with a simple mock. The mock provides enough of what is needed for inttest only, which is to generate some dummy content, but with proper file names. As a result, the increased timeout for the proto_gpb inttest should no longer be needed (inttest/proto_gpb/retest.config), both since the mock is now much faster to compile, and since any network bandwidth or connectivity issues have now been eliminated. --- inttest/proto_gpb/mock/gpb/src/gpb.app.src | 9 +++ inttest/proto_gpb/mock/gpb/src/gpb.erl | 4 + .../proto_gpb/mock/gpb/src/gpb_compile.erl | 73 +++++++++++++++++++ inttest/proto_gpb/rebar.config | 4 +- inttest/proto_gpb/retest.config | 5 -- 5 files changed, 87 insertions(+), 8 deletions(-) create mode 100644 inttest/proto_gpb/mock/gpb/src/gpb.app.src create mode 100644 inttest/proto_gpb/mock/gpb/src/gpb.erl create mode 100644 inttest/proto_gpb/mock/gpb/src/gpb_compile.erl delete mode 100644 inttest/proto_gpb/retest.config diff --git a/inttest/proto_gpb/mock/gpb/src/gpb.app.src b/inttest/proto_gpb/mock/gpb/src/gpb.app.src new file mode 100644 index 0000000..7225602 --- /dev/null +++ b/inttest/proto_gpb/mock/gpb/src/gpb.app.src @@ -0,0 +1,9 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +{application, gpb, + [{description, "Simple mock of gpb, with enough to generate dummy files"}, + {vsn, "1"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}]}. diff --git a/inttest/proto_gpb/mock/gpb/src/gpb.erl b/inttest/proto_gpb/mock/gpb/src/gpb.erl new file mode 100644 index 0000000..c694bbd --- /dev/null +++ b/inttest/proto_gpb/mock/gpb/src/gpb.erl @@ -0,0 +1,4 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(gpb). diff --git a/inttest/proto_gpb/mock/gpb/src/gpb_compile.erl b/inttest/proto_gpb/mock/gpb/src/gpb_compile.erl new file mode 100644 index 0000000..f7c4407 --- /dev/null +++ b/inttest/proto_gpb/mock/gpb/src/gpb_compile.erl @@ -0,0 +1,73 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2015 Tomas Abrahamsson (tomas.abrahamsson@gmail.com) +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- +-module(gpb_compile). +-export([file/2]). + +%% Simulate gpb compiling some proto files, +%% but generate only enough of what's needed for testing -- dummy stuff only. +file(Proto, Opts) -> + Prefix = proplists:get_value(module_name_prefix, Opts, ""), + Suffix = proplists:get_value(module_name_suffix, Opts, ""), + ProtoBase = filename:basename(Proto, ".proto"), + ModBase = Prefix ++ ProtoBase ++ Suffix, + ErlDest = filename:join(get_erl_outdir(Opts), ModBase ++ ".erl"), + HrlDest = filename:join(get_hrl_outdir(Opts), ModBase ++ ".hrl"), + ok = file:write_file(ErlDest, erl_text(ModBase)), + ok = file:write_file(HrlDest, hrl_text(ModBase)). + +erl_text(ModBase) -> + io_lib:format( + lines(["-module(~p).", + "-export([encode_msg/1]).", + "-export([decode_msg/2]).", + "", + "encode_msg(some_dummy_msg) -> <<1,2,3>>.", + "", + "decode_msg(<<1,2,3>>, _) -> some_dummy_msg."]), + [list_to_atom(ModBase)]). + +hrl_text(ModBase) -> + io_lib:format( + lines(["-ifndef(~s_hrl).", + "-define(~s_hrl, true).", + "", + "%% some record definitions would normally go here...", + "" + "-endif. %% ~s_hrl"]), + [ModBase, ModBase, ModBase]). + +get_erl_outdir(Opts) -> + proplists:get_value(o_erl, Opts, get_outdir(Opts)). + +get_hrl_outdir(Opts) -> + proplists:get_value(o_hrl, Opts, get_outdir(Opts)). + +get_outdir(Opts) -> + proplists:get_value(o, Opts, "."). + +lines(Lines) -> + lists:flatten([[L, $\n] || L <- Lines]). diff --git a/inttest/proto_gpb/rebar.config b/inttest/proto_gpb/rebar.config index a002245..06ac985 100644 --- a/inttest/proto_gpb/rebar.config +++ b/inttest/proto_gpb/rebar.config @@ -3,14 +3,12 @@ {erl_opts, [ - {i, "deps/gpb/include"}, {platform_define, "R13|R14", 'NO_CALLBACK_ATTRIBUTE'} ]}. {deps, [ - {gpb, [], {git, "git://github.com/tomas-abrahamsson/gpb", - {branch, "master"}}} + {gpb, [], {rsync, "../../../inttest/proto_gpb/mock/gpb"}} ]}. {proto_compiler, gpb}. diff --git a/inttest/proto_gpb/retest.config b/inttest/proto_gpb/retest.config deleted file mode 100644 index 01d5f9e..0000000 --- a/inttest/proto_gpb/retest.config +++ /dev/null @@ -1,5 +0,0 @@ -%% On my slow old netbook with a 1.6GHz CPU, this inttest -%% takes about 40 seconds to run, of which network activity -%% is about 3s. Double that to make avoid unnecessary timeouts -%% on e.g. loaded shared hosts, or with slow network connections. -{timeout, 90000}. From 2a0dd5bacfd34aeeb57115c9935d34b038302ef8 Mon Sep 17 00:00:00 2001 From: Tomas Abrahamsson Date: Fri, 2 Jan 2015 01:35:37 +0100 Subject: [PATCH 2/3] Bugfix comment in inttest/proto_protobuffs/rebar.config --- inttest/proto_protobuffs/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inttest/proto_protobuffs/rebar.config b/inttest/proto_protobuffs/rebar.config index 2158d17..6ed3900 100644 --- a/inttest/proto_protobuffs/rebar.config +++ b/inttest/proto_protobuffs/rebar.config @@ -15,4 +15,4 @@ %% The default proto compiler is protobuffs %% so don't need to specify this -%% {proto_compiler, gpb}. +%% {proto_compiler, protobuffs}. From 2af51af986b26b7f68ecb01377ca759e3b5cb536 Mon Sep 17 00:00:00 2001 From: Tomas Abrahamsson Date: Fri, 2 Jan 2015 01:36:47 +0100 Subject: [PATCH 3/3] Replace inttest external dep to protobuffs with a mock In inttest/proto_protobuffs, replace the external dependency to erlang_protobuffs, a protocol buffer compiler, with a simple mock. The mock provides enough of what is needed for inttest only, which is to generate some dummy content, but with proper file names. --- .../mock/protobuffs/src/protobuffs.app.src | 10 +++ .../mock/protobuffs/src/protobuffs.erl | 4 ++ .../protobuffs/src/protobuffs_compile.erl | 68 +++++++++++++++++++ inttest/proto_protobuffs/rebar.config | 4 +- 4 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 inttest/proto_protobuffs/mock/protobuffs/src/protobuffs.app.src create mode 100644 inttest/proto_protobuffs/mock/protobuffs/src/protobuffs.erl create mode 100644 inttest/proto_protobuffs/mock/protobuffs/src/protobuffs_compile.erl diff --git a/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs.app.src b/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs.app.src new file mode 100644 index 0000000..8017ba9 --- /dev/null +++ b/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs.app.src @@ -0,0 +1,10 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +{application, protobuffs, + [{description, + "Simple mock of erlang_protobuffs, with enough to generate dummy files"}, + {vsn, "1"}, + {registered, []}, + {applications, [kernel, stdlib]}, + {env, []}]}. diff --git a/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs.erl b/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs.erl new file mode 100644 index 0000000..614979d --- /dev/null +++ b/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs.erl @@ -0,0 +1,4 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et + +-module(protobuffs). diff --git a/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs_compile.erl b/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs_compile.erl new file mode 100644 index 0000000..9390dcd --- /dev/null +++ b/inttest/proto_protobuffs/mock/protobuffs/src/protobuffs_compile.erl @@ -0,0 +1,68 @@ +%% -*- erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 et +%% ------------------------------------------------------------------- +%% +%% rebar: Erlang Build Tools +%% +%% Copyright (c) 2015 Tomas Abrahamsson (tomas.abrahamsson@gmail.com) +%% +%% Permission is hereby granted, free of charge, to any person obtaining a copy +%% of this software and associated documentation files (the "Software"), to deal +%% in the Software without restriction, including without limitation the rights +%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +%% copies of the Software, and to permit persons to whom the Software is +%% furnished to do so, subject to the following conditions: +%% +%% The above copyright notice and this permission notice shall be included in +%% all copies or substantial portions of the Software. +%% +%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +%% THE SOFTWARE. +%% ------------------------------------------------------------------- +-module(protobuffs_compile). +-export([scan_file/2]). + +%% Simulate protobuffs compiling some proto files, +%% but generate only enough of what's needed for testing -- dummy stuff only. +scan_file(Proto, _Opts) -> + ProtoBase = filename:basename(Proto, ".proto"), + ModBase = ProtoBase ++ "_pb", + BeamDest = filename:join(get_beam_outdir(), ModBase ++ ".beam"), + HrlDest = filename:join(get_hrl_outdir(), ModBase ++ ".hrl"), + ok = file:write_file(BeamDest, beam_text(ModBase)), + ok = file:write_file(HrlDest, hrl_text(ModBase)). + +beam_text(ModBase) -> + Mod = list_to_atom(ModBase), + Forms = [mk_attr(module, Mod)], % just a -module(...). line + {ok, Mod, Bin} = compile:forms(Forms), + Bin. + +mk_attr(AttrName, AttrValue) -> + erl_syntax:revert( + erl_syntax:attribute(erl_syntax:atom(AttrName), + [erl_syntax:abstract(AttrValue)])). + +hrl_text(ModBase) -> + io_lib:format( + lines(["-ifndef(~s_hrl).", + "-define(~s_hrl, true).", + "", + "%% some record definitions would normally go here...", + "" + "-endif. %% ~s_hrl"]), + [ModBase, ModBase, ModBase]). + +get_beam_outdir() -> + ".". + +get_hrl_outdir() -> + ".". + +lines(Lines) -> + lists:flatten([[L, $\n] || L <- Lines]). diff --git a/inttest/proto_protobuffs/rebar.config b/inttest/proto_protobuffs/rebar.config index 6ed3900..86813f3 100644 --- a/inttest/proto_protobuffs/rebar.config +++ b/inttest/proto_protobuffs/rebar.config @@ -3,14 +3,12 @@ {erl_opts, [ - {i, "deps/protobuffs/include"}, {platform_define, "R13|R14", 'NO_CALLBACK_ATTRIBUTE'} ]}. {deps, [ - {protobuffs, [], {git, "git://github.com/basho/erlang_protobuffs", - {branch, "master"}}} + {protobuffs, [], {rsync, "../../../inttest/proto_protobuffs/mock/protobuffs"}} ]}. %% The default proto compiler is protobuffs