Support custom protobuf directory

proto_opts config option contains the compiler directive
that defines which proto compiler to use, also contains a
src_dirs entry that defines a list of locations for the
.proto files to be processed.

Add integration test that compiles .proto files from src
and from proto directory specified in separate rebar
config files
This commit is contained in:
Luis Rascao 2015-01-30 15:37:21 +00:00
parent 48e041dd9e
commit bf2b3e2468
11 changed files with 194 additions and 33 deletions

View file

@ -0,0 +1,19 @@
// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
// ex: ts=4 sw=4 et
package test3;
service test3
{
rpc testRpc3(RPC_INPUT3) returns (RPC_OUTPUT3);
}
message RPC_INPUT3
{
optional string str = 1;
}
message RPC_OUTPUT3
{
optional string str = 1;
}

View file

@ -0,0 +1,19 @@
// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
// ex: ts=4 sw=4 et
package test2;
service test2
{
rpc testRpc2(RPC_INPUT2) returns (RPC_OUTPUT2);
}
message RPC_INPUT2
{
optional string str = 1;
}
message RPC_OUTPUT2
{
optional string str = 1;
}

View file

@ -0,0 +1,19 @@
// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
// ex: ts=4 sw=4 et
package test5;
service test5
{
rpc testRpc5(RPC_INPUT5) returns (RPC_OUTPUT5);
}
message RPC_INPUT5
{
optional string str = 1;
}
message RPC_OUTPUT5
{
optional string str = 1;
}

View file

@ -0,0 +1,19 @@
// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
// ex: ts=4 sw=4 et
package test4;
service test4
{
rpc testRpc4(RPC_INPUT4) returns (RPC_OUTPUT4);
}
message RPC_INPUT4
{
optional string str = 1;
}
message RPC_OUTPUT4
{
optional string str = 1;
}

View file

@ -0,0 +1,19 @@
// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
// ex: ts=4 sw=4 et
package test;
service test
{
rpc testRpc(RPC_INPUT) returns (RPC_OUTPUT);
}
message RPC_INPUT
{
optional string str = 1;
}
message RPC_OUTPUT
{
optional string str = 1;
}

View file

@ -55,15 +55,29 @@ files() ->
[ [
{copy, "../../rebar", "rebar"}, {copy, "../../rebar", "rebar"},
{copy, "rebar.config", "rebar.config"}, {copy, "rebar.config", "rebar.config"},
{copy, "rebar2.config", "rebar2.config"},
{copy, "include", "include"}, {copy, "include", "include"},
{copy, "src", "src"}, {copy, "src", "src"},
{copy, "proto", "proto"},
{copy, "mock", "deps"}, {copy, "mock", "deps"},
{create, "ebin/foo.app", app(foo, ?MODULES ++ ?GENERATED_MODULES)} {create, "ebin/foo.app", app(foo, ?MODULES ++ ?GENERATED_MODULES)}
]. ].
run(_Dir) -> run(_Dir) ->
?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])), % perform test obtaining the .proto files from src dir
?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), ok = run_from_dir("src", "rebar.config"),
% perform test obtaining the .proto files from proto dir
ok = run_from_dir("proto", "rebar2.config").
run_from_dir(ProtoDir, ConfigFile) ->
?assertMatch({ok, _}, retest_sh:run("./rebar --config "
++ ConfigFile
++ " clean",
[])),
?assertMatch({ok, _}, retest_sh:run("./rebar --config "
++ ConfigFile
++ " compile",
[])),
%% Foo includes test_gpb.hrl, %% Foo includes test_gpb.hrl,
%% So if it compiled, that also means gpb succeeded in %% So if it compiled, that also means gpb succeeded in
%% generating the test_gpb.hrl file, and also that it generated %% generating the test_gpb.hrl file, and also that it generated
@ -72,21 +86,30 @@ run(_Dir) ->
?DEBUG("Verifying recompilation~n", []), ?DEBUG("Verifying recompilation~n", []),
TestErl = hd(generated_erl_files()), TestErl = hd(generated_erl_files()),
TestProto = hd(source_proto_files()), TestProto = hd(source_proto_files(ProtoDir)),
make_proto_newer_than_erl(TestProto, TestErl), make_proto_newer_than_erl(TestProto, TestErl),
TestMTime1 = read_mtime(TestErl), TestMTime1 = read_mtime(TestErl),
?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), ?assertMatch({ok, _}, retest_sh:run("./rebar --config "
++ ConfigFile
++ " compile",
[])),
TestMTime2 = read_mtime(TestErl), TestMTime2 = read_mtime(TestErl),
?assert(TestMTime2 > TestMTime1), ?assert(TestMTime2 > TestMTime1),
?DEBUG("Verifying recompilation with no changes~n", []), ?DEBUG("Verifying recompilation with no changes~n", []),
TestMTime3 = read_mtime(TestErl), TestMTime3 = read_mtime(TestErl),
?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), ?assertMatch({ok, _}, retest_sh:run("./rebar --config "
++ ConfigFile
++ " compile",
[])),
TestMTime4 = read_mtime(TestErl), TestMTime4 = read_mtime(TestErl),
?assert(TestMTime3 =:= TestMTime4), ?assert(TestMTime3 =:= TestMTime4),
?DEBUG("Verify cleanup~n", []), ?DEBUG("Verify cleanup~n", []),
?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])), ?assertMatch({ok, _}, retest_sh:run("./rebar --config "
++ ConfigFile
++ " clean",
[])),
ok = check_files_deleted(), ok = check_files_deleted(),
ok. ok.
@ -110,8 +133,8 @@ generated_hrl_files() ->
generated_beam_files() -> generated_beam_files() ->
add_dir("ebin", add_ext(?GENERATED_MODULES, ".beam")). add_dir("ebin", add_ext(?GENERATED_MODULES, ".beam")).
source_proto_files() -> source_proto_files(ProtoDir) ->
add_dir("src", ?SOURCE_PROTO_FILES). add_dir(ProtoDir, ?SOURCE_PROTO_FILES).
file_does_not_exist(F) -> file_does_not_exist(F) ->
not filelib:is_regular(F). not filelib:is_regular(F).

View file

@ -15,6 +15,8 @@
{gpb, ".*"} {gpb, ".*"}
]}. ]}.
{proto_compiler, gpb}. {proto_opts, [
{compiler, gpb}
]}.
{gpb_opts, [{module_name_suffix, "_gpb"}]}. {gpb_opts, [{module_name_suffix, "_gpb"}]}.

View file

@ -0,0 +1,23 @@
%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
%% ex: ts=4 sw=4 ft=erlang et
{erl_opts,
[
{platform_define, "R13|R14", 'NO_CALLBACK_ATTRIBUTE'}
]}.
{deps,
[
%% The dependency below to gpb is needed for "rebar compile" to
%% work, thus for the inttest to work, but the gpb that is actually
%% used in inttest is brought in from the inttest/proto_gpb/mock
%% subdirectory.
{gpb, ".*"}
]}.
{proto_opts, [
{compiler, gpb},
{src_dirs, ["proto"]}
]}.
{gpb_opts, [{module_name_suffix, "_gpb"}]}.

View file

@ -90,10 +90,15 @@
{erlydtl_opts, []}. {erlydtl_opts, []}.
%% == Proto compiler == %% == Proto compiler ==
{proto_compiler, protobuffs}. {proto_opts, [
{compiler, protobuffs},
{src_dirs, ["src"]}
]}.
%% Available compilers for protocol buffer files (*.proto): %% Available compilers for protocol buffer files (*.proto):
%% protobuffs (default) %% protobuffs (default)
%% gpb %% gpb
%% Optional src_dirs which is a list of directories where
%% to look for .proto files, default is src
%% Options for the gpb protocol buffer compiler, %% Options for the gpb protocol buffer compiler,
%% if selected by the proto_compiler option %% if selected by the proto_compiler option

View file

@ -43,26 +43,34 @@
%% Public API %% Public API
%% =================================================================== %% ===================================================================
find_proto_files(ProtoDirs) ->
lists:foldl(fun(ProtoDir, Acc) ->
rebar_utils:find_files_by_ext(ProtoDir, ".proto") ++ Acc
end,
[], ProtoDirs).
compile(Config, AppFile) -> compile(Config, AppFile) ->
case rebar_utils:find_files_by_ext("src", ".proto") of %% Find a compiler for protocol buffers,
%% use that for compiling protocol buffers
{CompilerModule, ProtoDirs} = select_proto_compiler_and_dir(Config),
case find_proto_files(ProtoDirs) of
[] -> [] ->
ok; ok;
Protos -> Protos ->
%% Find a compiler for protocol buffers, %% Ask the proto compiler to compile the .proto files.
%% use that for compiling protocol buffers
CompilerModule = select_proto_compiler(Config),
CompilerModule:proto_compile(Config, AppFile, Protos) CompilerModule:proto_compile(Config, AppFile, Protos)
end. end.
clean(Config, AppFile) -> clean(Config, AppFile) ->
%% Find a compiler for protocol buffers,
%% use that for clean protocol buffers
{CompilerModule, ProtoDirs} = select_proto_compiler_and_dir(Config),
%% Get a list of generated .beam and .hrl files and then delete them %% Get a list of generated .beam and .hrl files and then delete them
Protos = rebar_utils:find_files_by_ext("src", ".proto"), case find_proto_files(ProtoDirs) of
case Protos of
[] -> [] ->
ok; ok;
_ -> Protos ->
%% Ask the proto compiler to compile the .proto files. %% Ask the proto compiler to clean the .proto files.
CompilerModule = select_proto_compiler(Config),
CompilerModule:proto_clean(Config, AppFile, Protos) CompilerModule:proto_clean(Config, AppFile, Protos)
end. end.
@ -80,7 +88,10 @@ info_help(GeneralDescr, Cmd) ->
"~s.~n" "~s.~n"
++ "~n" ++ "~n"
++ "Valid rebar.config options:~n" ++ "Valid rebar.config options:~n"
++ " {proto_compiler, Compiler}~n" ++ " {proto_opts, [~n"
++ " {compiler, Compiler},~n"
++ " {src_dirs, [Dir]}~n"
++ " ]}~n"
++ "The following protocol buffer compilers are available:~n" ++ "The following protocol buffer compilers are available:~n"
++ "~s~n", ++ "~s~n",
[GeneralDescr, format_proto_compiler_list()]), [GeneralDescr, format_proto_compiler_list()]),
@ -118,19 +129,22 @@ is_proto_compiler_module(Module) ->
false false
end. end.
select_proto_compiler(Config) -> select_proto_compiler_and_dir(Config) ->
Default = get_default_compiler(), Default = get_default_compiler(),
Key = rebar_config:get_local(Config, proto_compiler, Default), ProtoOpts = rebar_config:get_local(Config, proto_opts, []),
Key = proplists:get_value(compiler, ProtoOpts, Default),
ProtoDirs = proplists:get_value(src_dirs, ProtoOpts, ["src"]),
AvailCompilers = find_proto_compilers(), AvailCompilers = find_proto_compilers(),
case lists:keyfind(Key, #proto_compiler.key, AvailCompilers) of CompilerModule = case lists:keyfind(Key, #proto_compiler.key, AvailCompilers) of
#proto_compiler{module=CompilerModule} -> #proto_compiler{module=Module} ->
CompilerModule; Module;
false -> false ->
?ABORT("No such protocol buffer compiler known, '~s'~n" ?ABORT("No such protocol buffer compiler known, '~s'~n"
++ "The following are known:~n" ++ "The following are known:~n"
++ "~s~n", ++ "~s~n",
[Key, format_proto_compiler_list()]) [Key, format_proto_compiler_list()])
end. end,
{CompilerModule, ProtoDirs}.
format_proto_compiler_list() -> format_proto_compiler_list() ->
Default = get_default_compiler(), Default = get_default_compiler(),

View file

@ -42,13 +42,12 @@
key() -> key() ->
gpb. gpb.
proto_compile(Config, _AppFile, _ProtoFiles) -> proto_compile(Config, _AppFile, Files) ->
%% Check for gpb library -- if it's not present, fail %% Check for gpb library -- if it's not present, fail
%% since we have.proto files that need building %% since we have.proto files that need building
case gpb_is_present() of case gpb_is_present() of
true -> true ->
GpbOpts = user_gpb_opts(Config), GpbOpts = user_gpb_opts(Config),
Files = rebar_utils:find_files_by_ext("src", ".proto"),
Targets = [filename:join("src", target_filename(F, GpbOpts)) Targets = [filename:join("src", target_filename(F, GpbOpts))
|| F <- Files], || F <- Files],
rebar_base_compiler:run(Config, [], rebar_base_compiler:run(Config, [],