Recompile .proto files with gpb also with prefix/suffix

For protocol buffer files, when there were gpb options to alter the
module name with prefix or suffix, recompilation was not properly
detected. This is now fixed. (Issue #384).

Properly detecting change meant moving from rebar_base_compiler, so it
also meant processing protocol buffer files in sequence instead of in
parallel.

inttest/proto_gpb/proto_gpb_rt.erl

@@ -29,6 +29,8 @@
          run/1]).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("kernel/include/file.hrl").
+-include_lib("deps/retest/include/retest.hrl").
 
 -define(MODULES,
         [foo,
@@ -42,6 +44,13 @@
          test4_gpb,
          test5_gpb]).
 
+-define(SOURCE_PROTO_FILES,
+        ["test.proto",
+         "a/test2.proto",
+         "a/b/test3.proto",
+         "c/test4.proto",
+         "c/d/test5.proto"]).
+
 files() ->
     [
      {copy, "../../rebar", "rebar"},
@@ -60,6 +69,17 @@ run(_Dir) ->
     %% generating the test_gpb.hrl file, and also that it generated
     %% the .hrl file was generated before foo was compiled.
     ok = check_beams_generated(),
+
+    ?DEBUG("Verifying recompilation~n", []),
+    TestErl = hd(generated_erl_files()),
+    TestProto = hd(source_proto_files()),
+    make_proto_newer_than_erl(TestProto, TestErl),
+    TestMTime1 = read_mtime(TestErl),
+    ?assertMatch({ok, _}, retest_sh:run("./rebar compile", [])),
+    TestMTime2 = read_mtime(TestErl),
+    ?assert(TestMTime2 > TestMTime1),
+
+    ?DEBUG("Verify cleanup~n", []),
     ?assertMatch({ok, _}, retest_sh:run("./rebar clean", [])),
     ok = check_files_deleted(),
     ok.
@@ -81,6 +101,12 @@ generated_erl_files() ->
 generated_hrl_files() ->
     add_dir("include", add_ext(?GENERATED_MODULES, ".hrl")).
 
+generated_beam_files() ->
+    add_dir("ebin", add_ext(?GENERATED_MODULES, ".beam")).
+
+source_proto_files() ->
+    add_dir("src", ?SOURCE_PROTO_FILES).
+
 file_does_not_exist(F) ->
     not filelib:is_regular(F).
 
@@ -90,6 +116,30 @@ add_ext(Modules, Ext) ->
 add_dir(Dir, Files) ->
     [filename:join(Dir, File) || File <- Files].
 
+read_mtime(File) ->
+    {ok, #file_info{mtime=MTime}} = file:read_file_info(File),
+    MTime.
+
+
+make_proto_newer_than_erl(Proto, Erl) ->
+    %% Do this by back-dating the erl file instead of touching the
+    %% proto file.  Do this instead of sleeping for a second to get a
+    %% reliable test.  Sleeping would have been needed sin ce the
+    %% #file_info{} (used by eg. filelib:last_modified) does not have
+    %% sub-second resolution (even though most file systems have).
+    {ok, #file_info{mtime=ProtoMTime}} = file:read_file_info(Proto),
+    {ok, ErlInfo} = file:read_file_info(Erl),
+    OlderMTime = update_seconds_to_datetime(ProtoMTime, -2),
+    OlderErlInfo = ErlInfo#file_info{mtime = OlderMTime},
+    ok = file:write_file_info(Erl, OlderErlInfo).
+
+update_seconds_to_datetime(DT, ToAdd) ->
+    calendar:gregorian_seconds_to_datetime(
+      calendar:datetime_to_gregorian_seconds(DT) + ToAdd).
+
+touch_file(File) ->
+    ?assertMatch({ok, _}, retest_sh:run("touch " ++ File, [])).
+
 check(Check, Files) ->
     lists:foreach(
       fun(F) ->

src/rebar_proto_gpb_compiler.erl

@@ -42,29 +42,36 @@
 key() ->
     gpb.
 
-proto_compile(Config, _AppFile, _ProtoFiles) ->
+proto_compile(Config, _AppFile, ProtoFiles) ->
     %% Check for gpb library -- if it's not present, fail
     %% since we have.proto files that need building
     case gpb_is_present() of
         true ->
-            rebar_base_compiler:run(Config, [],
+            UserGpbOpts = user_gpb_opts(Config),
-                                    "src", ".proto",
+            lists:foreach(
-                                    "src", ".erl",
+              fun(ProtoFile) ->
-                                    fun compile_gpb/3,
+                      GpbOpts = UserGpbOpts ++ default_dest_opts()
-                                    [{check_last_mod, true}]);
+                          ++ default_include_opts(ProtoFile),
+
+                      case needs_compile(ProtoFile, GpbOpts) of
+                          true ->
+                              compile_gpb(ProtoFile, GpbOpts);
+                          false ->
+                              ok
+                      end
+              end,
+              ProtoFiles);
         false ->
             ?ERROR("The gpb library is not present in code path!\n", []),
             ?FAIL
     end.
 
 proto_clean(Config, _AppFile, ProtoFiles) ->
-    GpbOpts = gpb_opts(Config),
+    GpbOpts = user_gpb_opts(Config) ++ default_dest_opts(),
-    MPrefix = proplists:get_value(module_name_prefix, GpbOpts, ""),
-    MSuffix = proplists:get_value(module_name_suffix, GpbOpts, ""),
     rebar_file_utils:delete_each(
-      [beam_relpath(MPrefix, F, MSuffix) || F <- ProtoFiles]
+      [beam_file(F, GpbOpts) || F <- ProtoFiles]
-      ++ [erl_relpath(MPrefix, F, MSuffix) || F <- ProtoFiles]
+      ++ [erl_file(F, GpbOpts) || F <- ProtoFiles]
-      ++ [hrl_relpath(MPrefix, F, MSuffix) || F <- ProtoFiles]),
+      ++ [hrl_file(F, GpbOpts) || F <- ProtoFiles]),
     ok.
 
 %% ===================================================================
@@ -82,17 +89,27 @@ proto_info(help, compile) ->
 proto_info(help, clean) ->
     ?CONSOLE("", []).
 
-gpb_opts(Config) ->
-    rebar_config:get_local(Config, gpb_opts, []).
-
 gpb_is_present() ->
     code:which(gpb) =/= non_existing.
 
-compile_gpb(Source, _Target, Config) ->
+user_gpb_opts(Config) ->
+    rebar_config:get_local(Config, gpb_opts, []).
+
+default_dest_opts() ->
+    [{o_erl, "src"}, {o_hrl, "include"}].
+
+default_include_opts(Source) ->
+    SourceFullPath = filename:absname(Source),
+    [{i,filename:dirname(SourceFullPath)}].
+
+needs_compile(ProtoFile, GpbOpts) ->
+    Erl = erl_file(ProtoFile, GpbOpts),
+    Hrl = hrl_file(ProtoFile, GpbOpts),
+    filelib:last_modified(Erl) < filelib:last_modified(ProtoFile) orelse
+        filelib:last_modified(Hrl) < filelib:last_modified(ProtoFile).
+
+compile_gpb(Source, GpbOpts) ->
     SourceFullPath = filename:absname(Source),
-    DefaultDestOpts = [{o_erl, "src"}, {o_hrl, "include"}],
-    SelfIncludeOpt = [{i,filename:dirname(SourceFullPath)}],
-    GpbOpts = gpb_opts(Config) ++ DefaultDestOpts ++ SelfIncludeOpt,
     ok = filelib:ensure_dir(filename:join("ebin", "dummy")),
     ok = filelib:ensure_dir(filename:join("include", "dummy")),
     ?CONSOLE("Compiling ~s\n", [Source]),
@@ -104,16 +121,28 @@ compile_gpb(Source, _Target, Config) ->
             ?FAIL
     end.
 
-beam_relpath(Prefix, Proto, Suffix) ->
+beam_file(ProtoFile, GpbOpts) ->
-    proto_filename_to_relpath("ebin", Prefix, Proto, Suffix, ".beam").
+    proto_filename_to_path("ebin", ProtoFile, ".beam", GpbOpts).
 
-erl_relpath(Prefix, Proto, Suffix) ->
+erl_file(ProtoFile, GpbOpts) ->
-    proto_filename_to_relpath("src", Prefix, Proto, Suffix, ".erl").
+    ErlOutDir = get_erl_outdir(GpbOpts),
+    proto_filename_to_path(ErlOutDir, ProtoFile, ".erl", GpbOpts).
 
-hrl_relpath(Prefix, Proto, Suffix) ->
+hrl_file(ProtoFile, GpbOpts) ->
-    proto_filename_to_relpath("include", Prefix, Proto, Suffix, ".hrl").
+    HrlOutDir = get_hrl_outdir(GpbOpts),
+    proto_filename_to_path(HrlOutDir, ProtoFile, ".hrl", GpbOpts).
 
-proto_filename_to_relpath(Dir, Prefix, Proto, Suffix, NewExt) ->
+proto_filename_to_path(Dir, ProtoFile, NewExt, GpbOpts) ->
-    BaseNoExt = filename:basename(Proto, ".proto"),
+    BaseNoExt = filename:basename(ProtoFile, ".proto"),
+    Prefix = proplists:get_value(module_name_prefix, GpbOpts, ""),
+    Suffix = proplists:get_value(module_name_suffix, GpbOpts, ""),
     filename:join([Dir, Prefix ++ BaseNoExt ++ Suffix ++ NewExt]).
 
+get_erl_outdir(Opts) ->
+    proplists:get_value(o_erl, Opts, get_outdir(Opts)).
+
+get_hrl_outdir(Opts) ->
+    proplists:get_value(o_hrl, Opts, get_outdir(Opts)).
+
+get_outdir(Opts) ->
+    proplists:get_value(o, Opts, ".").