Refactor of compilation logic to enable support for package modules and provide better modularity/reusability for other systems

ebin/rebar.app

@@ -3,7 +3,8 @@
   {vsn, "2"},
   {modules, [ rebar,
               rebar_app_utils,
-              rebar_config,              
+              rebar_base_compiler,
+              rebar_config,
               rebar_core,
               rebar_ct,
               rebar_deps,

src/rebar_base_compiler.erl

@@ -0,0 +1,197 @@
+%% -*- tab-width: 4;erlang-indent-level: 4;indent-tabs-mode: nil -*-
+%% ex: ts=4 sw=4 et
+%% -------------------------------------------------------------------
+%%
+%% rebar: Erlang Build Tools
+%%
+%% Copyright (c) 2009 Dave Smith (dizzyd@dizzyd.com)
+%%
+%% Permission is hereby granted, free of charge, to any person obtaining a copy
+%% of this software and associated documentation files (the "Software"), to deal
+%% in the Software without restriction, including without limitation the rights
+%% to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the Software is
+%% furnished to do so, subject to the following conditions:
+%%
+%% The above copyright notice and this permission notice shall be included in
+%% all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+%% IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+%% FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+%% AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+%% LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+%% OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+%% THE SOFTWARE.
+%% -------------------------------------------------------------------
+-module(rebar_base_compiler).
+
+-include("rebar.hrl").
+
+-export([run/8]).
+
+
+%% ===================================================================
+%% Public API
+%% ===================================================================
+
+run(Config, SourceDir, SourceExt, TargetDir, TargetExt,
+    FirstFiles, CompileFn, Opts) ->
+    SourceExtRe = ".*\\" ++ SourceExt ++ [$$],
+
+    %% Options:
+    %% recurse_source_dir
+    %% needs_compile_checks - [ fun/2 ]
+    Recursive = proplists:get_bool(recurse_source_dir, Opts),
+
+    %% Find all the source files we can
+    FoundFiles = filelib:fold_files(SourceDir, SourceExtRe, Recursive,
+                                 fun(F, Acc) -> [F | Acc] end, []),
+
+    %% Construct two lists of targets. "FirstTargets" is the list of files which
+    %% must be compiled first and in strict order; "RestTargets" is all remaining files
+    %% that may be compiled in any order.
+    FirstTargets = [{Fs, target_file(Fs, SourceDir, SourceExt, TargetDir, TargetExt)} ||
+                       Fs <- FirstFiles],
+    RestTargets = [{Fs, target_file(Fs, SourceDir, SourceExt, TargetDir, TargetExt)} ||
+                      Fs <- drop_each(FirstFiles, FoundFiles)],
+
+    %% Setup list of functions which determine if a file needs compilation or not. By
+    %% default we just check the last modified date
+    NeedsCompileFns = [ fun check_source_lastmod/3 ] ++
+        rebar_config:get(Config, needs_compile_checks, []),
+
+    %% Compile the first targets in sequence
+    compile_each(FirstTargets, Config, NeedsCompileFns, CompileFn),
+
+    %% Spin up workers
+    case RestTargets of
+        [] ->
+            ok;
+        _ ->
+            Self = self(),
+            F = fun() -> compile_worker(Self, Config, NeedsCompileFns, CompileFn) end,
+            Pids = [spawn_monitor(F) || _I <- lists:seq(1,3)],
+            compile_queue(Pids, RestTargets)
+    end.
+
+
+%% ===================================================================
+%% Internal functions
+%% ===================================================================
+
+target_file(SourceFile, SourceDir, SourceExt, TargetDir, TargetExt) ->
+    %% Remove all leading components of the source dir from the file -- we want
+    %% to maintain the deeper structure (if any) of the source file path
+    BaseFile = remove_common_path(SourceFile, SourceDir),
+    filename:join([TargetDir, filename:dirname(BaseFile),
+                   filename:basename(BaseFile, SourceExt) ++ TargetExt]).
+
+
+remove_common_path(Fname, Path) ->
+    remove_common_path1(filename:split(Fname), filename:split(Path)).
+
+remove_common_path1([Part | RestFilename], [Part | RestPath]) ->
+    remove_common_path1(RestFilename, RestPath);
+remove_common_path1(FilenameParts, _) ->
+    filename:join(FilenameParts).
+
+
+drop_each([], List) ->
+    List;
+drop_each([Member | Rest], List) ->
+    drop_each(Rest, lists:delete(Member, List)).
+
+
+needs_compile(_SourceFile, _TargetFile, _Config, []) ->
+    false;
+needs_compile(SourceFile, TargetFile, Config, [Fn | Rest]) ->
+    case Fn(SourceFile, TargetFile, Config) of
+        true ->
+            true;
+        false ->
+            needs_compile(SourceFile, TargetFile, Config, Rest)
+    end.
+
+check_source_lastmod(SourceFile, TargetFile, _Config) ->
+    filelib:last_modified(TargetFile) < filelib:last_modified(SourceFile).
+
+compile(Source, Target, Config, NeedsCompileFns, CompileFn) ->
+    case needs_compile(Source, Target, Config, NeedsCompileFns) of
+        true ->
+            ok = filelib:ensure_dir(Target),
+            CompileFn(Source, Target, Config);
+        false ->
+            skipped
+    end.
+
+
+
+compile_each([], _Config, _NeedsCompileFns, _CompileFn) ->
+    ok;
+compile_each([{Source, Target} | Rest], Config, NeedsCompileFns, CompileFn) ->
+    case compile(Source, Target, Config, NeedsCompileFns, CompileFn) of
+        ok ->
+            ?CONSOLE("Compiled ~s\n", [Source]);
+        skipped ->
+            ?INFO("Skipped ~s\n", [Source])
+    end,
+    compile_each(Rest, Config, NeedsCompileFns, CompileFn).
+
+
+
+compile_queue([], []) ->
+    ok;
+compile_queue(Pids, Targets) ->
+    receive
+        {next, Worker} ->
+            case Targets of
+                [] ->
+                    Worker ! empty,
+                    compile_queue(Pids, Targets);
+                [{Source, Target} | Rest] ->
+                    Worker ! {compile, Source, Target},
+                    compile_queue(Pids, Rest)
+            end;
+
+        {fail, Error} ->
+            ?DEBUG("Worker compilation failed: ~p\n", [Error]),
+            ?FAIL;
+
+        {compiled, Source} ->
+            ?CONSOLE("Compiled ~s\n", [Source]),
+            compile_queue(Pids, Targets);
+
+        {skipped, Source} ->
+            ?INFO("Skipped ~s\n", [Source]),
+            compile_queue(Pids, Targets);
+
+        {'DOWN', Mref, _, Pid, normal} ->
+            ?DEBUG("Worker exited cleanly\n", []),
+            Pids2 = lists:delete({Pid, Mref}, Pids),
+            compile_queue(Pids2, Targets);
+
+        {'DOWN', _Mref, _, _Pid, Info} ->
+            ?DEBUG("Worker failed: ~p\n", [Info]),
+            ?FAIL
+    end.
+
+compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn) ->
+    QueuePid ! {next, self()},
+    receive
+        {compile, Source, Target} ->
+            case catch(compile(Source, Target, Config, NeedsCompileFns, CompileFn)) of
+                ok ->
+                    QueuePid ! {compiled, Source},
+                    compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn);
+                skipped ->
+                    QueuePid ! {skipped, Source},
+                    compile_worker(QueuePid, Config, NeedsCompileFns, CompileFn);
+                Error ->
+                    QueuePid ! {fail, Error},
+                    ok
+            end;
+
+        empty ->
+            ok
+    end.

src/rebar_erlc_compiler.erl

@@ -30,7 +30,7 @@
          clean/2]).
 
  %% make available for rebar_eunit until there is a better option
--export([do_compile/8, compile_opts/2, list_hrls/2]).
+-export([hrls_check/3]).
 
 -include("rebar.hrl").
 
@@ -39,18 +39,30 @@
 %% ===================================================================
 
 compile(Config, _AppFile) ->
-    do_compile(Config, "src/*.erl", "ebin", ".erl", ".beam",
+    rebar_base_compiler:run(Config, "src", ".erl", "ebin", ".beam",
-               fun list_hrls/2, fun compile_erl/2,
+                            rebar_config:get_list(Config, erl_first_files, []),
-               rebar_config:get_list(Config, erl_first_files, [])),
+                            fun compile_erl/3,
-    do_compile(Config, "mibs/*.mib", "priv/mibs", ".mib", ".bin",
+                            [recurse_source_dir,
-               undefined, fun compile_mib/2,
+                             {needs_compile_checks, [fun hrls_check/3]}]),
-               rebar_config:get_list(Config, mib_first_files, [])).
+
+    rebar_base_compiler:run(Config, "mibs", ".mib", "priv/mibs", ".bin",
+                            rebar_config:get_list(Config, mib_first_files, []),
+                            fun compile_mib/3,
+                            []).
 
 clean(_Config, _AppFile) ->
     %% TODO: This would be more portable if it used Erlang to traverse
     %%       the dir structure and delete each file; however it would also
     %%       much slower.
-    [] = os:cmd("rm -f ebin/*.beam priv/mibs/*.bin"),
+    ok = rebar_file_utils:rm_rf("ebin/*.beam priv/mibs/*.bin"),
+
+    %% Erlang compilation is recursive, so it's possible that we have a nested
+    %% directory structure in ebin with .beam files within. As such, we want
+    %% to scan whatever is left in the ebin/ directory for sub-dirs which
+    %% satisfy our criteria. TODO: Is there a better way to do this?
+    BeamFiles = filelib:fold_files("ebin", "^.*\\.beam\$", true,
+                                   fun(F, BeamFiles) -> BeamFiles ++ [F] end, []),
+    rebar_file_utils:delete_each(BeamFiles),
     ok.
 
 
@@ -59,71 +71,11 @@ clean(_Config, _AppFile) ->
 %% Internal functions
 %% ===================================================================
 
-do_compile(Config, SrcWildcard, OutDir, InExt, OutExt,
+hrls_check(Source, Target, Config) ->
-           IncludeFn, CompileFn, FirstFiles) ->
+    TargetLastMod = filelib:last_modified(Target),
-    case filelib:wildcard(SrcWildcard) of
+    lists:any(fun(I) -> TargetLastMod < filelib:last_modified(I) end,
-        [] ->
+              list_hrls(Source, Config)).
-            ok;
-        FoundFiles when is_list(FoundFiles) ->
-            %% Ensure that the FirstFiles are compiled first; drop them from the
-            %% FoundFiles and compile them in sequence
-            FirstTargets = [{Fs, target_file(Fs, OutDir, InExt, OutExt)} || Fs <- FirstFiles],
-            RestTargets = [{Fs, target_file(Fs, OutDir, InExt, OutExt)} ||
-                              Fs <- drop_each(FirstFiles, FoundFiles)],
 
-            %% Make sure target directory exists
-            ok = filelib:ensure_dir(target_file(hd(FoundFiles), OutDir, InExt, OutExt)),
-
-            %% Compile first targets in sequence
-            compile_each(FirstTargets, Config, IncludeFn, CompileFn),
-
-            %% Spin up workers
-            Self = self(),
-            Pids = [spawn_monitor(fun() -> compile_worker(Self) end) || _I <- lists:seq(1,3)],
-
-            %% Process rest of targets
-            compile_queue(Pids, RestTargets, Config, IncludeFn, CompileFn)
-    end.
-
-drop_each([], List) ->
-    List;
-drop_each([Member | Rest], List) ->
-    drop_each(Rest, lists:delete(Member, List)).
-
-compile_each([], _Config, _IncludeFn, _CompileFn) ->
-    ok;
-compile_each([{Src, Target} | Rest], Config, IncludeFn, CompileFn) ->
-    case needs_compile(Src, Target, IncludeFn, Config) of
-        true ->
-            ?CONSOLE("Compiling ~s\n", [Src]),
-            CompileFn(Src, Config);
-        false ->
-            ?INFO("Skipping ~s\n", [Src]),
-            ok
-    end,
-    compile_each(Rest, Config, IncludeFn, CompileFn).
-
-needs_compile(Src, Target, IncludeFn, Config) ->
-    TargetLM = filelib:last_modified(Target),
-    case TargetLM < filelib:last_modified(Src) of
-        true ->
-            true;
-        false ->
-            if is_function(IncludeFn) ->
-                    lists:any(fun(I) ->
-                                      TargetLM < filelib:last_modified(I)
-                              end,
-                              IncludeFn(Src, Config));
-               true ->
-                    false
-            end
-    end.
-
-target_file(F, TargetDir, InExt, OutExt) ->
-    filename:join([TargetDir, filename:basename(F, InExt) ++ OutExt]).
-
-compile_opts(Config, Key) ->
-    rebar_config:get_list(Config, Key, []).
 
 list_hrls(Src, Config) ->
     case epp:open(Src, include_path(Src, Config)) of
@@ -134,26 +86,31 @@ list_hrls(Src, Config) ->
             false
     end.
 
-include_path(Src, Config) ->
-    [filename:dirname(Src)|compile_opts(Config, i)].
 
 extract_includes(Epp, Src) ->
     case epp:parse_erl_form(Epp) of
-	{ok, {attribute, 1, file, {Src, 1}}} ->
+        {ok, {attribute, 1, file, {Src, 1}}} ->
-	    extract_includes(Epp, Src);
+            extract_includes(Epp, Src);
-	{ok, {attribute, 1, file, {IncFile, 1}}} ->
+        {ok, {attribute, 1, file, {IncFile, 1}}} ->
             [IncFile|extract_includes(Epp, Src)];
-	{ok, _} ->
+        {ok, _} ->
-	    extract_includes(Epp, Src);
+            extract_includes(Epp, Src);
-	{eof, _} ->
+        {eof, _} ->
-	    epp:close(Epp),
+            epp:close(Epp),
-	    [];
+            [];
-	{error, _Error} ->
+        {error, _Error} ->
-	    extract_includes(Epp, Src)
+            extract_includes(Epp, Src)
     end.
 
-compile_erl(Source, Config) ->
+include_path(Source, Config) ->
-    Opts = [{i, "include"}, {outdir, "ebin"}, report, return] ++ compile_opts(Config, erl_opts),
+    [filename:dirname(Source) | compile_opts(Config, i)].
+
+compile_opts(Config, Key) ->
+    rebar_config:get_list(Config, Key, []).
+
+compile_erl(Source, Target, Config) ->
+    Opts = [{i, "include"}, {outdir, filename:dirname(Target)}, report, return] ++
+            compile_opts(Config, erl_opts),
     case compile:file(Source, Opts) of
         {ok, _, []} ->
             ok;
@@ -169,7 +126,7 @@ compile_erl(Source, Config) ->
             ?FAIL
     end.
 
-compile_mib(Source, Config) ->
+compile_mib(Source, _Target, Config) ->
     Opts = [{outdir, "priv/mibs"}, {i, ["priv/mibs"]}] ++ compile_opts(Config, mib_opts),
     case snmpc:compile(Source, Opts) of
         {ok, _} ->
@@ -177,58 +134,3 @@ compile_mib(Source, Config) ->
         {error, compilation_failed} ->
             ?FAIL
     end.
-
-compile_queue([], [], _Config, _IncludeFn, _CompileFn) ->
-    ok;
-compile_queue(Pids, Targets, Config, IncludeFn, CompileFn) ->
-    receive
-        {next, Worker} ->
-            case Targets of
-                [] ->
-                    Worker ! empty,
-                    compile_queue(Pids, Targets, Config, IncludeFn, CompileFn);
-                [{Src, Target} | Rest] ->
-                    Worker ! {compile, Src, Target, Config, IncludeFn, CompileFn},
-                    compile_queue(Pids, Rest, Config, IncludeFn, CompileFn)
-            end;
-
-        {fail, Error} ->
-            ?DEBUG("Worker compilation failed: ~p\n", [Error]),
-            ?FAIL;
-
-        {compiled, Source} ->
-            ?CONSOLE("Compiled ~s\n", [Source]),
-            compile_queue(Pids, Targets, Config, IncludeFn, CompileFn);
-
-        {'DOWN', Mref, _, Pid, normal} ->
-            ?DEBUG("Worker exited cleanly\n", []),
-            Pids2 = lists:delete({Pid, Mref}, Pids),
-            compile_queue(Pids2, Targets, Config, IncludeFn, CompileFn);
-
-        {'DOWN', _Mref, _, _Pid, Info} ->
-            ?DEBUG("Worker failed: ~p\n", [Info]),
-            ?FAIL
-    end.
-
-compile_worker(QueuePid) ->
-    QueuePid ! {next, self()},
-    receive
-        {compile, Src, Target, Config, IncludeFn, CompileFn} ->
-            case needs_compile(Src, Target, IncludeFn, Config) of
-                true ->
-                    case catch(CompileFn(Src, Config)) of
-                        ok ->
-                            QueuePid ! {compiled, Src},
-                            compile_worker(QueuePid);
-                        Error ->
-                            QueuePid ! {fail, Error},
-                            ok
-                    end;
-                false ->
-                    ?INFO("Skipping ~s\n", [Src]),
-                    compile_worker(QueuePid)
-            end;
-
-        empty ->
-            ok
-    end.

src/rebar_erlydtl_compiler.erl

@@ -82,14 +82,10 @@
 
 compile(Config, _AppFile) ->
     DtlOpts = erlydtl_opts(Config),
-    rebar_erlc_compiler:do_compile(
+    rebar_base_compiler:run(Config, option(doc_root, DtlOpts), option(source_ext, DtlOpts),
-      Config,
+                            option(out_dir, DtlOpts), option(module_ext, DtlOpts) ++ ".beam",
-      filename:join([option(doc_root, DtlOpts),
+                            [], fun compile_dtl/3,
-                     ["*",option(source_ext, DtlOpts)]]),
+                            [{needs_compile_checks, [fun referenced_dtls/3]}]).
-      option(out_dir, DtlOpts),
-      option(source_ext, DtlOpts),
-      option(module_ext, DtlOpts)++".beam",
-      fun referenced_dtls/2, fun compile_dtl/2, []).
 
 
 %% ===================================================================
@@ -107,13 +103,13 @@ default(out_dir)  -> "ebin";
 default(source_ext) -> ".dtl";
 default(module_ext) -> "_dtl".
 
-referenced_dtls(Source, Config) ->
+referenced_dtls(Source, _Target, Config) ->
-    Set = referenced_dtls([Source], Config,
+    Set = referenced_dtls1([Source], Config,
                           sets:add_element(Source, sets:new())),
     Final = sets:to_list(sets:del_element(Source, Set)),
     Final.
 
-referenced_dtls(Step, Config, Seen) ->
+referenced_dtls1(Step, Config, Seen) ->
     DtlOpts = erlydtl_opts(Config),
     ExtMatch = re:replace(option(source_ext, DtlOpts), "\.", "\\\\.",
                           [{return, list}]),
@@ -132,7 +128,7 @@ referenced_dtls(Step, Config, Seen) ->
                              sets:union(New, Seen))
     end.
 
-compile_dtl(Source, Config) ->
+compile_dtl(Source, _Target, Config) ->
     case code:which(erlydtl) of
         non_existing ->
             ?CONSOLE(

src/rebar_eunit.erl

@@ -52,15 +52,19 @@ eunit(Config, _File) ->
     ok = filelib:ensure_dir(?EUNIT_DIR ++ "/foo"),
 
     %% Compile all erlang from src/ into ?EUNIT_DIR
-    rebar_erlc_compiler:do_compile(Config, "src/*.erl", ?EUNIT_DIR, ".erl", ".beam",
+    rebar_base_compiler:run(Config, "src", ".erl", ?EUNIT_DIR, ".beam",
-                                   fun rebar_erlc_compiler:list_hrls/2, fun compile_erl/2,
+                            rebar_config:get_list(Config, erl_first_files, []),
-                                   rebar_config:get_list(Config, erl_first_files, [])),
+                            fun compile_erl/3,
+                            [recurse_source_dir,
+                             {needs_compile_checks, [fun rebar_erlc_compiler:hrls_check/3]}]),
 
     %% Build a list of all the .beams in ?EUNIT_DIR -- use this for cover
     %% and eunit testing. Normally you can just tell cover and/or eunit to
     %% scan the directory for you, but eunit does a code:purge in conjunction
     %% with that scan and causes any cover compilation info to be lost. So,
     %% we do it by hand. :(
+    %%
+    %% TODO: Not currently compatible with package modules
     Modules = [list_to_atom(filename:basename(N, ".beam")) ||
                   N <- filelib:wildcard("*.beam", "ebin")],
 
@@ -123,7 +127,7 @@ clean(_Config, _File) ->
 %% Internal functions
 %% ===================================================================
 
-compile_erl(Source, Config) ->
+compile_erl(Source, Target, Config) ->
     case is_quickcheck_avail() of
         true ->
             EqcOpts = [{d, 'EQC'}];
@@ -133,7 +137,8 @@ compile_erl(Source, Config) ->
 
     ErlOpts = rebar_config:get_list(Config, erl_opts, []),
     EunitOpts = rebar_config:get_list(Config, eunit_compile_opts, []),
-    Opts = [{i, "include"}, {outdir, ?EUNIT_DIR}, {d, 'TEST'}, debug_info, report] ++
+    Opts = [{i, "include"}, {outdir, filename:dirname(Target)},
+            {d, 'TEST'}, debug_info, report] ++
         ErlOpts ++ EunitOpts ++ EqcOpts,
     case compile:file(Source, Opts) of
         {ok, _} ->

src/rebar_lfe_compiler.erl

@@ -38,15 +38,16 @@
 
 compile(Config, _AppFile) ->
     FirstFiles = rebar_config:get_list(Config, lfe_first_files, []),
-    rebar_erlc_compiler:do_compile(Config, "src/*.lfe", "ebin", ".lfe", ".beam",
+    rebar_base_compiler:run(Config, "src", ".lfe", "ebin", ".beam",
-                                   undefined, fun compile_lfe/2, FirstFiles).
+                            FirstFiles,
+                            fun compile_lfe/3, []).
 
 
 %% ===================================================================
 %% Internal functions
 %% ===================================================================
 
-compile_lfe(Source, Config) ->
+compile_lfe(Source, _Target, Config) ->
     case code:which(lfe_comp) of
         non_existing ->
             ?CONSOLE("~n===============================================~n" ++

src/rebar_otp_app.erl

@@ -119,7 +119,8 @@ validate_name(AppName, File) ->
         true ->
             ok;
         false ->
-            ?ERROR("Invalid ~s: name of application (~p) must match filename.\n", [File, AppName]),
+            ?ERROR("Invalid ~s: name of application (~p) must match filename.\n",
+                   [File, AppName]),
             ?FAIL
     end.
 
@@ -130,7 +131,7 @@ validate_modules(AppName, undefined) ->
 validate_modules(AppName, Mods) ->
     %% Construct two sets -- one for the actual .beam files in ebin/ and one for the modules
     %% listed in the .app file
-    EbinSet = ordsets:from_list([beam_to_mod(N) || N <- filelib:wildcard("ebin/*.beam")]),
+    EbinSet = ordsets:from_list([beam_to_mod(N) || N <- beams()]),
     ModSet = ordsets:from_list(Mods),
 
     %% Identify .beam files listed in the .app, but not present in ebin/
@@ -156,5 +157,10 @@ validate_modules(AppName, Mods) ->
     end.
 
 beam_to_mod(Filename) ->
-    list_to_atom(filename:basename(Filename, ".beam")).
+    ["ebin" | Rest] = filename:split(Filename),
+    list_to_atom(filename:basename(string:join(Rest, "."), ".beam")).
+
+beams() ->
+    filelib:fold_files("ebin", ".*\.beam\$", true,
+                       fun(F, Acc) -> [F | Acc] end, []).