Support custom protobuf directory

proto_opts config option contains the compiler directive
that defines which proto compiler to use, also contains a
src_dirs entry that defines a list of locations for the
.proto files to be processed.

Add integration test that compiles .proto files from src
and from proto directory specified in separate rebar
config files

inttest/proto_gpb/proto/a/b/test3.proto

@@ -0,0 +1,19 @@
+// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
+// ex: ts=4 sw=4 et
+
+package test3;
+
+service test3
+{
+    rpc testRpc3(RPC_INPUT3) returns (RPC_OUTPUT3);
+}
+
+message RPC_INPUT3
+{
+    optional string str = 1;
+}
+
+message RPC_OUTPUT3
+{
+    optional string str = 1;
+}

inttest/proto_gpb/proto/a/test2.proto

@@ -0,0 +1,19 @@
+// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
+// ex: ts=4 sw=4 et
+
+package test2;
+
+service test2
+{
+    rpc testRpc2(RPC_INPUT2) returns (RPC_OUTPUT2);
+}
+
+message RPC_INPUT2
+{
+    optional string str = 1;
+}
+
+message RPC_OUTPUT2
+{
+    optional string str = 1;
+}

inttest/proto_gpb/proto/c/d/test5.proto

@@ -0,0 +1,19 @@
+// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
+// ex: ts=4 sw=4 et
+
+package test5;
+
+service test5
+{
+    rpc testRpc5(RPC_INPUT5) returns (RPC_OUTPUT5);
+}
+
+message RPC_INPUT5
+{
+    optional string str = 1;
+}
+
+message RPC_OUTPUT5
+{
+    optional string str = 1;
+}

inttest/proto_gpb/proto/c/test4.proto

@@ -0,0 +1,19 @@
+// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
+// ex: ts=4 sw=4 et
+
+package test4;
+
+service test4
+{
+    rpc testRpc4(RPC_INPUT4) returns (RPC_OUTPUT4);
+}
+
+message RPC_INPUT4
+{
+    optional string str = 1;
+}
+
+message RPC_OUTPUT4
+{
+    optional string str = 1;
+}

inttest/proto_gpb/proto/test.proto

@@ -0,0 +1,19 @@
+// -*- c-basic-offset: 4; indent-tabs-mode: nil -*-
+// ex: ts=4 sw=4 et
+
+package test;
+
+service test
+{
+    rpc testRpc(RPC_INPUT) returns (RPC_OUTPUT);
+}
+
+message RPC_INPUT
+{
+    optional string str = 1;
+}
+
+message RPC_OUTPUT
+{
+    optional string str = 1;
+}

inttest/proto_gpb/proto_gpb_rt.erl

@@ -55,15 +55,29 @@ files() ->
     [
      {copy, "../../rebar", "rebar"},
      {copy, "rebar.config", "rebar.config"},
+     {copy, "rebar2.config", "rebar2.config"},
      {copy, "include", "include"},
      {copy, "src", "src"},
+     {copy, "proto", "proto"},
      {copy, "mock", "deps"},
      {create, "ebin/foo.app", app(foo, ?MODULES ++ ?GENERATED_MODULES)}
     ].
 
 run(_Dir) ->
-    ?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])),
+    % perform test obtaining the .proto files from src dir
-    ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])),
+    ok = run_from_dir("src", "rebar.config"),
+    % perform test obtaining the .proto files from proto dir
+    ok = run_from_dir("proto", "rebar2.config").
+
+run_from_dir(ProtoDir, ConfigFile) ->
+    ?assertMatch({ok, _}, retest_sh:run("./rebar --config "
+                                        ++ ConfigFile
+                                        ++ " clean",
+                                        [])),
+    ?assertMatch({ok, _}, retest_sh:run("./rebar --config "
+                                        ++ ConfigFile
+                                        ++ " compile",
+                                        [])),
     %% Foo includes test_gpb.hrl,
     %% So if it compiled, that also means gpb succeeded in
     %% generating the test_gpb.hrl file, and also that it generated
@@ -72,21 +86,30 @@ run(_Dir) ->
 
     ?DEBUG("Verifying recompilation~n", []),
     TestErl = hd(generated_erl_files()),
-    TestProto = hd(source_proto_files()),
+    TestProto = hd(source_proto_files(ProtoDir)),
     make_proto_newer_than_erl(TestProto, TestErl),
     TestMTime1 = read_mtime(TestErl),
-    ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])),
+    ?assertMatch({ok, _}, retest_sh:run("./rebar --config "
+                                        ++ ConfigFile
+                                        ++ " compile",
+                                        [])),
     TestMTime2 = read_mtime(TestErl),
     ?assert(TestMTime2 > TestMTime1),
 
     ?DEBUG("Verifying recompilation with no changes~n", []),
     TestMTime3 = read_mtime(TestErl),
-    ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])),
+    ?assertMatch({ok, _}, retest_sh:run("./rebar --config "
+                                        ++ ConfigFile
+                                        ++ " compile",
+                                        [])),
     TestMTime4 = read_mtime(TestErl),
     ?assert(TestMTime3 =:= TestMTime4),
 
     ?DEBUG("Verify cleanup~n", []),
-    ?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])),
+    ?assertMatch({ok, _}, retest_sh:run("./rebar --config "
+                                        ++ ConfigFile
+                                        ++ " clean",
+                                        [])),
     ok = check_files_deleted(),
     ok.
 
@@ -110,8 +133,8 @@ generated_hrl_files() ->
 generated_beam_files() ->
     add_dir("ebin", add_ext(?GENERATED_MODULES, ".beam")).
 
-source_proto_files() ->
+source_proto_files(ProtoDir) ->
-    add_dir("src", ?SOURCE_PROTO_FILES).
+    add_dir(ProtoDir, ?SOURCE_PROTO_FILES).
 
 file_does_not_exist(F) ->
     not filelib:is_regular(F).

inttest/proto_gpb/rebar.config

@@ -15,6 +15,8 @@
   {gpb, ".*"}
  ]}.
 
-{proto_compiler, gpb}.
+{proto_opts, [
+    {compiler, gpb}
+]}.
 
 {gpb_opts, [{module_name_suffix, "_gpb"}]}.

inttest/proto_gpb/rebar2.config

@@ -0,0 +1,23 @@
+%% -*- mode: erlang;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 ft=erlang et
+
+{erl_opts,
+ [
+  {platform_define, "R13|R14", 'NO_CALLBACK_ATTRIBUTE'}
+ ]}.
+
+{deps,
+ [
+  %% The dependency below to gpb is needed for "rebar compile" to
+  %% work, thus for the inttest to work, but the gpb that is actually
+  %% used in inttest is brought in from the inttest/proto_gpb/mock
+  %% subdirectory.
+  {gpb, ".*"}
+ ]}.
+
+{proto_opts, [
+    {compiler, gpb},
+    {src_dirs, ["proto"]}
+]}.
+
+{gpb_opts, [{module_name_suffix, "_gpb"}]}.

rebar.config.sample

@@ -90,10 +90,15 @@
 {erlydtl_opts, []}.
 
 %% == Proto compiler ==
-{proto_compiler, protobuffs}.
+{proto_opts, [
+    {compiler, protobuffs},
+    {src_dirs, ["src"]}
+]}.
 %% Available compilers for protocol buffer files (*.proto):
 %%   protobuffs  (default)
 %%   gpb
+%% Optional src_dirs which is a list of directories where
+%% to look for .proto files, default is src
 
 %% Options for the gpb protocol buffer compiler,
 %% if selected by the proto_compiler option

src/rebar_proto_compiler.erl

@@ -43,26 +43,34 @@
 %% Public API
 %% ===================================================================
 
+find_proto_files(ProtoDirs) ->
+    lists:foldl(fun(ProtoDir, Acc) ->
+                  rebar_utils:find_files_by_ext(ProtoDir, ".proto") ++ Acc
+                end,
+                [], ProtoDirs).
+
 compile(Config, AppFile) ->
-    case rebar_utils:find_files_by_ext("src", ".proto") of
+    %% Find a compiler for protocol buffers,
+    %% use that for compiling protocol buffers
+    {CompilerModule, ProtoDirs} = select_proto_compiler_and_dir(Config),
+    case find_proto_files(ProtoDirs) of
         [] ->
             ok;
         Protos ->
-            %% Find a compiler for protocol buffers,
+            %% Ask the proto compiler to compile the .proto files.
-            %% use that for compiling protocol buffers
-            CompilerModule = select_proto_compiler(Config),
             CompilerModule:proto_compile(Config, AppFile, Protos)
     end.
 
 clean(Config, AppFile) ->
+    %% Find a compiler for protocol buffers,
+    %% use that for clean protocol buffers
+    {CompilerModule, ProtoDirs} = select_proto_compiler_and_dir(Config),
     %% Get a list of generated .beam and .hrl files and then delete them
-    Protos = rebar_utils:find_files_by_ext("src", ".proto"),
+    case find_proto_files(ProtoDirs) of
-    case Protos of
         [] ->
             ok;
-        _ ->
+        Protos ->
-            %% Ask the proto compiler to compile the .proto files.
+            %% Ask the proto compiler to clean the .proto files.
-            CompilerModule = select_proto_compiler(Config),
             CompilerModule:proto_clean(Config, AppFile, Protos)
     end.
 
@@ -80,7 +88,10 @@ info_help(GeneralDescr, Cmd) ->
        "~s.~n"
        ++ "~n"
        ++ "Valid rebar.config options:~n"
-       ++ "  {proto_compiler, Compiler}~n"
+       ++ " {proto_opts, [~n"
+       ++ "   {compiler, Compiler},~n"
+       ++ "   {src_dirs, [Dir]}~n"
+       ++ " ]}~n"
        ++ "The following protocol buffer compilers are available:~n"
        ++ "~s~n",
        [GeneralDescr, format_proto_compiler_list()]),
@@ -118,19 +129,22 @@ is_proto_compiler_module(Module) ->
             false
     end.
 
-select_proto_compiler(Config) ->
+select_proto_compiler_and_dir(Config) ->
     Default = get_default_compiler(),
-    Key = rebar_config:get_local(Config, proto_compiler, Default),
+    ProtoOpts = rebar_config:get_local(Config, proto_opts, []),
+    Key = proplists:get_value(compiler, ProtoOpts, Default),
+    ProtoDirs = proplists:get_value(src_dirs, ProtoOpts, ["src"]),
     AvailCompilers = find_proto_compilers(),
-    case lists:keyfind(Key, #proto_compiler.key, AvailCompilers) of
+    CompilerModule = case lists:keyfind(Key, #proto_compiler.key, AvailCompilers) of
-        #proto_compiler{module=CompilerModule} ->
+                      #proto_compiler{module=Module} ->
-            CompilerModule;
+                          Module;
                       false ->
                           ?ABORT("No such protocol buffer compiler known, '~s'~n"
                                  ++ "The following are known:~n"
                                  ++ "~s~n",
                                  [Key, format_proto_compiler_list()])
-    end.
+                     end,
+    {CompilerModule, ProtoDirs}.
 
 format_proto_compiler_list() ->
     Default = get_default_compiler(),

src/rebar_proto_gpb_compiler.erl

@@ -42,13 +42,12 @@
 key() ->
     gpb.
 
-proto_compile(Config, _AppFile, _ProtoFiles) ->
+proto_compile(Config, _AppFile, Files) ->
     %% Check for gpb library -- if it's not present, fail
     %% since we have.proto files that need building
     case gpb_is_present() of
         true ->
             GpbOpts = user_gpb_opts(Config),
-            Files = rebar_utils:find_files_by_ext("src", ".proto"),
             Targets = [filename:join("src", target_filename(F, GpbOpts))
                        || F <- Files],
             rebar_base_compiler:run(Config, [],