From bf2b3e246821e5697b6154ce0d2254600c91b34c Mon Sep 17 00:00:00 2001 From: Luis Rascao Date: Fri, 30 Jan 2015 15:37:21 +0000 Subject: [PATCH] Support custom protobuf directory proto_opts config option contains the compiler directive that defines which proto compiler to use, also contains a src_dirs entry that defines a list of locations for the .proto files to be processed. Add integration test that compiles .proto files from src and from proto directory specified in separate rebar config files --- inttest/proto_gpb/proto/a/b/test3.proto | 19 +++++++++ inttest/proto_gpb/proto/a/test2.proto | 19 +++++++++ inttest/proto_gpb/proto/c/d/test5.proto | 19 +++++++++ inttest/proto_gpb/proto/c/test4.proto | 19 +++++++++ inttest/proto_gpb/proto/test.proto | 19 +++++++++ inttest/proto_gpb/proto_gpb_rt.erl | 39 +++++++++++++---- inttest/proto_gpb/rebar.config | 4 +- inttest/proto_gpb/rebar2.config | 23 ++++++++++ rebar.config.sample | 7 +++- src/rebar_proto_compiler.erl | 56 +++++++++++++++---------- src/rebar_proto_gpb_compiler.erl | 3 +- 11 files changed, 194 insertions(+), 33 deletions(-) create mode 100644 inttest/proto_gpb/proto/a/b/test3.proto create mode 100644 inttest/proto_gpb/proto/a/test2.proto create mode 100644 inttest/proto_gpb/proto/c/d/test5.proto create mode 100644 inttest/proto_gpb/proto/c/test4.proto create mode 100644 inttest/proto_gpb/proto/test.proto create mode 100644 inttest/proto_gpb/rebar2.config diff --git a/inttest/proto_gpb/proto/a/b/test3.proto b/inttest/proto_gpb/proto/a/b/test3.proto new file mode 100644 index 0000000..6e3372f --- /dev/null +++ b/inttest/proto_gpb/proto/a/b/test3.proto @@ -0,0 +1,19 @@ +// -*- c-basic-offset: 4; indent-tabs-mode: nil -*- +// ex: ts=4 sw=4 et + +package test3; + +service test3 +{ + rpc testRpc3(RPC_INPUT3) returns (RPC_OUTPUT3); +} + +message RPC_INPUT3 +{ + optional string str = 1; +} + +message RPC_OUTPUT3 +{ + optional string str = 1; +} diff --git a/inttest/proto_gpb/proto/a/test2.proto b/inttest/proto_gpb/proto/a/test2.proto new file mode 100644 index 0000000..6a2d1ac --- /dev/null +++ b/inttest/proto_gpb/proto/a/test2.proto @@ -0,0 +1,19 @@ +// -*- c-basic-offset: 4; indent-tabs-mode: nil -*- +// ex: ts=4 sw=4 et + +package test2; + +service test2 +{ + rpc testRpc2(RPC_INPUT2) returns (RPC_OUTPUT2); +} + +message RPC_INPUT2 +{ + optional string str = 1; +} + +message RPC_OUTPUT2 +{ + optional string str = 1; +} diff --git a/inttest/proto_gpb/proto/c/d/test5.proto b/inttest/proto_gpb/proto/c/d/test5.proto new file mode 100644 index 0000000..e94b3bc --- /dev/null +++ b/inttest/proto_gpb/proto/c/d/test5.proto @@ -0,0 +1,19 @@ +// -*- c-basic-offset: 4; indent-tabs-mode: nil -*- +// ex: ts=4 sw=4 et + +package test5; + +service test5 +{ + rpc testRpc5(RPC_INPUT5) returns (RPC_OUTPUT5); +} + +message RPC_INPUT5 +{ + optional string str = 1; +} + +message RPC_OUTPUT5 +{ + optional string str = 1; +} diff --git a/inttest/proto_gpb/proto/c/test4.proto b/inttest/proto_gpb/proto/c/test4.proto new file mode 100644 index 0000000..3e1de74 --- /dev/null +++ b/inttest/proto_gpb/proto/c/test4.proto @@ -0,0 +1,19 @@ +// -*- c-basic-offset: 4; indent-tabs-mode: nil -*- +// ex: ts=4 sw=4 et + +package test4; + +service test4 +{ + rpc testRpc4(RPC_INPUT4) returns (RPC_OUTPUT4); +} + +message RPC_INPUT4 +{ + optional string str = 1; +} + +message RPC_OUTPUT4 +{ + optional string str = 1; +} diff --git a/inttest/proto_gpb/proto/test.proto b/inttest/proto_gpb/proto/test.proto new file mode 100644 index 0000000..9b3cf59 --- /dev/null +++ b/inttest/proto_gpb/proto/test.proto @@ -0,0 +1,19 @@ +// -*- c-basic-offset: 4; indent-tabs-mode: nil -*- +// ex: ts=4 sw=4 et + +package test; + +service test +{ + rpc testRpc(RPC_INPUT) returns (RPC_OUTPUT); +} + +message RPC_INPUT +{ + optional string str = 1; +} + +message RPC_OUTPUT +{ + optional string str = 1; +} diff --git a/inttest/proto_gpb/proto_gpb_rt.erl b/inttest/proto_gpb/proto_gpb_rt.erl index 263e9db..09fcde2 100644 --- a/inttest/proto_gpb/proto_gpb_rt.erl +++ b/inttest/proto_gpb/proto_gpb_rt.erl @@ -55,15 +55,29 @@ files() -> [ {copy, "../../rebar", "rebar"}, {copy, "rebar.config", "rebar.config"}, + {copy, "rebar2.config", "rebar2.config"}, {copy, "include", "include"}, {copy, "src", "src"}, + {copy, "proto", "proto"}, {copy, "mock", "deps"}, {create, "ebin/foo.app", app(foo, ?MODULES ++ ?GENERATED_MODULES)} ]. run(_Dir) -> - ?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])), - ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), + % perform test obtaining the .proto files from src dir + ok = run_from_dir("src", "rebar.config"), + % perform test obtaining the .proto files from proto dir + ok = run_from_dir("proto", "rebar2.config"). + +run_from_dir(ProtoDir, ConfigFile) -> + ?assertMatch({ok, _}, retest_sh:run("./rebar --config " + ++ ConfigFile + ++ " clean", + [])), + ?assertMatch({ok, _}, retest_sh:run("./rebar --config " + ++ ConfigFile + ++ " compile", + [])), %% Foo includes test_gpb.hrl, %% So if it compiled, that also means gpb succeeded in %% generating the test_gpb.hrl file, and also that it generated @@ -72,21 +86,30 @@ run(_Dir) -> ?DEBUG("Verifying recompilation~n", []), TestErl = hd(generated_erl_files()), - TestProto = hd(source_proto_files()), + TestProto = hd(source_proto_files(ProtoDir)), make_proto_newer_than_erl(TestProto, TestErl), TestMTime1 = read_mtime(TestErl), - ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), + ?assertMatch({ok, _}, retest_sh:run("./rebar --config " + ++ ConfigFile + ++ " compile", + [])), TestMTime2 = read_mtime(TestErl), ?assert(TestMTime2 > TestMTime1), ?DEBUG("Verifying recompilation with no changes~n", []), TestMTime3 = read_mtime(TestErl), - ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])), + ?assertMatch({ok, _}, retest_sh:run("./rebar --config " + ++ ConfigFile + ++ " compile", + [])), TestMTime4 = read_mtime(TestErl), ?assert(TestMTime3 =:= TestMTime4), ?DEBUG("Verify cleanup~n", []), - ?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])), + ?assertMatch({ok, _}, retest_sh:run("./rebar --config " + ++ ConfigFile + ++ " clean", + [])), ok = check_files_deleted(), ok. @@ -110,8 +133,8 @@ generated_hrl_files() -> generated_beam_files() -> add_dir("ebin", add_ext(?GENERATED_MODULES, ".beam")). -source_proto_files() -> - add_dir("src", ?SOURCE_PROTO_FILES). +source_proto_files(ProtoDir) -> + add_dir(ProtoDir, ?SOURCE_PROTO_FILES). file_does_not_exist(F) -> not filelib:is_regular(F). diff --git a/inttest/proto_gpb/rebar.config b/inttest/proto_gpb/rebar.config index b2f558f..7526aa0 100644 --- a/inttest/proto_gpb/rebar.config +++ b/inttest/proto_gpb/rebar.config @@ -15,6 +15,8 @@ {gpb, ".*"} ]}. -{proto_compiler, gpb}. +{proto_opts, [ + {compiler, gpb} +]}. {gpb_opts, [{module_name_suffix, "_gpb"}]}. diff --git a/inttest/proto_gpb/rebar2.config b/inttest/proto_gpb/rebar2.config new file mode 100644 index 0000000..7f6bfb7 --- /dev/null +++ b/inttest/proto_gpb/rebar2.config @@ -0,0 +1,23 @@ +%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*- +%% ex: ts=4 sw=4 ft=erlang et + +{erl_opts, + [ + {platform_define, "R13|R14", 'NO_CALLBACK_ATTRIBUTE'} + ]}. + +{deps, + [ + %% The dependency below to gpb is needed for "rebar compile" to + %% work, thus for the inttest to work, but the gpb that is actually + %% used in inttest is brought in from the inttest/proto_gpb/mock + %% subdirectory. + {gpb, ".*"} + ]}. + +{proto_opts, [ + {compiler, gpb}, + {src_dirs, ["proto"]} +]}. + +{gpb_opts, [{module_name_suffix, "_gpb"}]}. diff --git a/rebar.config.sample b/rebar.config.sample index 107a02f..91d3dff 100644 --- a/rebar.config.sample +++ b/rebar.config.sample @@ -90,10 +90,15 @@ {erlydtl_opts, []}. %% == Proto compiler == -{proto_compiler, protobuffs}. +{proto_opts, [ + {compiler, protobuffs}, + {src_dirs, ["src"]} +]}. %% Available compilers for protocol buffer files (*.proto): %% protobuffs (default) %% gpb +%% Optional src_dirs which is a list of directories where +%% to look for .proto files, default is src %% Options for the gpb protocol buffer compiler, %% if selected by the proto_compiler option diff --git a/src/rebar_proto_compiler.erl b/src/rebar_proto_compiler.erl index 2d3eb2b..f98526e 100644 --- a/src/rebar_proto_compiler.erl +++ b/src/rebar_proto_compiler.erl @@ -43,26 +43,34 @@ %% Public API %% =================================================================== +find_proto_files(ProtoDirs) -> + lists:foldl(fun(ProtoDir, Acc) -> + rebar_utils:find_files_by_ext(ProtoDir, ".proto") ++ Acc + end, + [], ProtoDirs). + compile(Config, AppFile) -> - case rebar_utils:find_files_by_ext("src", ".proto") of + %% Find a compiler for protocol buffers, + %% use that for compiling protocol buffers + {CompilerModule, ProtoDirs} = select_proto_compiler_and_dir(Config), + case find_proto_files(ProtoDirs) of [] -> ok; Protos -> - %% Find a compiler for protocol buffers, - %% use that for compiling protocol buffers - CompilerModule = select_proto_compiler(Config), + %% Ask the proto compiler to compile the .proto files. CompilerModule:proto_compile(Config, AppFile, Protos) end. clean(Config, AppFile) -> + %% Find a compiler for protocol buffers, + %% use that for clean protocol buffers + {CompilerModule, ProtoDirs} = select_proto_compiler_and_dir(Config), %% Get a list of generated .beam and .hrl files and then delete them - Protos = rebar_utils:find_files_by_ext("src", ".proto"), - case Protos of + case find_proto_files(ProtoDirs) of [] -> ok; - _ -> - %% Ask the proto compiler to compile the .proto files. - CompilerModule = select_proto_compiler(Config), + Protos -> + %% Ask the proto compiler to clean the .proto files. CompilerModule:proto_clean(Config, AppFile, Protos) end. @@ -80,7 +88,10 @@ info_help(GeneralDescr, Cmd) -> "~s.~n" ++ "~n" ++ "Valid rebar.config options:~n" - ++ " {proto_compiler, Compiler}~n" + ++ " {proto_opts, [~n" + ++ " {compiler, Compiler},~n" + ++ " {src_dirs, [Dir]}~n" + ++ " ]}~n" ++ "The following protocol buffer compilers are available:~n" ++ "~s~n", [GeneralDescr, format_proto_compiler_list()]), @@ -118,19 +129,22 @@ is_proto_compiler_module(Module) -> false end. -select_proto_compiler(Config) -> +select_proto_compiler_and_dir(Config) -> Default = get_default_compiler(), - Key = rebar_config:get_local(Config, proto_compiler, Default), + ProtoOpts = rebar_config:get_local(Config, proto_opts, []), + Key = proplists:get_value(compiler, ProtoOpts, Default), + ProtoDirs = proplists:get_value(src_dirs, ProtoOpts, ["src"]), AvailCompilers = find_proto_compilers(), - case lists:keyfind(Key, #proto_compiler.key, AvailCompilers) of - #proto_compiler{module=CompilerModule} -> - CompilerModule; - false -> - ?ABORT("No such protocol buffer compiler known, '~s'~n" - ++ "The following are known:~n" - ++ "~s~n", - [Key, format_proto_compiler_list()]) - end. + CompilerModule = case lists:keyfind(Key, #proto_compiler.key, AvailCompilers) of + #proto_compiler{module=Module} -> + Module; + false -> + ?ABORT("No such protocol buffer compiler known, '~s'~n" + ++ "The following are known:~n" + ++ "~s~n", + [Key, format_proto_compiler_list()]) + end, + {CompilerModule, ProtoDirs}. format_proto_compiler_list() -> Default = get_default_compiler(), diff --git a/src/rebar_proto_gpb_compiler.erl b/src/rebar_proto_gpb_compiler.erl index 37f901c..c528a4a 100644 --- a/src/rebar_proto_gpb_compiler.erl +++ b/src/rebar_proto_gpb_compiler.erl @@ -42,13 +42,12 @@ key() -> gpb. -proto_compile(Config, _AppFile, _ProtoFiles) -> +proto_compile(Config, _AppFile, Files) -> %% Check for gpb library -- if it's not present, fail %% since we have.proto files that need building case gpb_is_present() of true -> GpbOpts = user_gpb_opts(Config), - Files = rebar_utils:find_files_by_ext("src", ".proto"), Targets = [filename:join("src", target_filename(F, GpbOpts)) || F <- Files], rebar_base_compiler:run(Config, [],