Heavy-duty refactor to support truly transitive dependencies

--HG--
extra : rebase_source : 41c7f1c337a7cb63582aecd7b87ba998b40ba3aa

src/rebar_core.erl

@@ -28,8 +28,6 @@
 
 -export([run/1]).
 
--export([app_dir/1, rel_dir/1]). % Ugh
-
 -include("rebar.hrl").
 
 
@@ -71,8 +69,11 @@ run(RawArgs) ->
     rebar_config:set_global(escript, filename:absname(escript:script_name())),
     ?DEBUG("Rebar location: ~p\n", [rebar_config:get_global(escript, undefined)]),
 
+    %% Note the top-level directory for reference
+    rebar_config:set_global(base_dir, filename:absname(rebar_utils:get_cwd())),
+
     %% Load rebar.config, if it exists
-    [process_dir(rebar_utils:get_cwd(), rebar_config:new(), Command)
+    [process_dir(rebar_utils:get_cwd(), rebar_config:new(), Command, sets:new())
      || Command <- CommandAtoms],
     ok.
 
@@ -250,13 +251,14 @@ filter_flags([Item | Rest], Commands) ->
     end.
 
 
-process_dir(Dir, ParentConfig, Command) ->
+process_dir(Dir, ParentConfig, Command, DirSet) ->
     case filelib:is_dir(Dir) of
         false ->
             ?WARN("Skipping non-existent sub-dir: ~p\n", [Dir]),
-            ok;
+            DirSet;
 
         true ->
+            ?DEBUG("Entering ~s\n", [Dir]),
             ok = file:set_cwd(Dir),
             Config = rebar_config:new(ParentConfig),
 
@@ -272,151 +274,92 @@ process_dir(Dir, ParentConfig, Command) ->
             {ok, AvailModuleSets} = application:get_env(rebar, modules),
             {DirModules, ModuleSetFile} = choose_module_set(AvailModuleSets, Dir),
 
-            %% Get the list of modules for "any dir". This is a catch-all list of modules
-            %% that are processed in addition to modules associated with this directory
-            %% type. These any_dir modules are processed FIRST.
+            %% Get the list of modules for "any dir". This is a catch-all list
+            %% of modules that are processed in addition to modules associated
+            %% with this directory type. These any_dir modules are processed
+            %% FIRST.
             {ok, AnyDirModules} = application:get_env(rebar, any_dir_modules),
 
             Modules = AnyDirModules ++ DirModules,
 
-            ok = process_subdirs(Dir, Modules, Config, ModuleSetFile, Command),
+            %% Invoke 'preprocess' on the modules -- this yields a list of other
+            %% directories that should be processed _before_ the current one.
+            Predirs = acc_modules(Modules, preprocess, Config, ModuleSetFile),
+            ?DEBUG("Predirs: ~p\n", [Predirs]),
+            DirSet2 = process_each(Predirs, Command, Config, ModuleSetFile, DirSet),
+
+            %% Make sure the CWD is reset properly; processing the dirs may have
+            %% caused it to change
+            ok = file:set_cwd(Dir),
+
+            %% Execute the current command on this directory
+            execute(Command, Modules, Config, ModuleSetFile),
+
+            %% Mark the current directory as processed
+            DirSet3 = sets:add_element(Dir, DirSet2),
+
+            %% Invoke 'postprocess' on the modules -- this yields a list of other
+            %% directories that should be processed _after_ the current one.
+            Postdirs = acc_modules(Modules, postprocess, Config, ModuleSetFile),
+            ?DEBUG("Postdirs: ~p\n", [Postdirs]),
+            DirSet4 = process_each(Postdirs, Command, Config, ModuleSetFile, DirSet3),
+
+            %% Make sure the CWD is reset properly; processing the dirs may have
+            %% caused it to change
+            ok = file:set_cwd(Dir),
 
             %% Once we're all done processing, reset the code path to whatever
             %% the parent initialized it to
             restore_code_path(CurrentCodePath),
-            ok
+
+            %% Return the updated dirset as our result
+            DirSet4
     end.
 
 
 %%
-%% Run the preprocessors and execute the command on all newly
-%% found Dirs until no new Dirs are found by the preprocessors.
+%% Given a list of directories and a set of previously processed directories,
+%% process each one we haven't seen yet
 %%
-process_subdirs(Dir, Modules, Config, ModuleSetFile, Command) ->
-    process_subdirs(Dir, Modules, Config, ModuleSetFile, Command, sets:new()).
-
-process_subdirs(Dir, Modules, Config, ModuleSetFile, Command, ProcessedDirs) ->
-    %% Give the modules a chance to tweak config and indicate if there
-    %% are any other dirs that might need processing first.
-    {UpdatedConfig, Dirs} = acc_modules(Modules, preprocess, Config, ModuleSetFile),
-    ?DEBUG("~s subdirs: ~p\n", [Dir, Dirs]),
-
-    %% Add ebin to path if this app has any plugins configured locally.
-    prep_plugin_modules(UpdatedConfig),
-
-    %% Process subdirs that haven't already been processed.
-    F = fun (D, S) ->
-                case filelib:is_dir(D) andalso (not sets:is_element(D, S)) of
+process_each([], _Command, _Config, _ModuleSetFile, DirSet) ->
+    DirSet;
+process_each([Dir | Rest], Command, Config, ModuleSetFile, DirSet) ->
+    case sets:is_element(Dir, DirSet) of
         true ->
-                        process_dir(D, UpdatedConfig, Command),
-                        sets:add_element(D, S);
+            ?DEBUG("Skipping ~s; already processed!\n", [Dir]),
+            process_each(Rest, Command, Config, ModuleSetFile, DirSet);
         false ->
-                        S
-                end
-        end,
-    NewProcessedDirs = lists:foldl(F, sets:add_element(parent, ProcessedDirs), Dirs),
-
-    %% Make sure the CWD is reset properly; processing subdirs may have caused it
-    %% to change
-    ok = file:set_cwd(Dir),
-
-    %% Run the parent commands exactly once as well
-    case sets:is_element(parent, ProcessedDirs) of
-        true ->
-            ok;
-        false ->
-            %% Get the list of plug-in modules from rebar.config. These modules are
-            %% processed LAST and do not participate in preprocess.
-            {ok, PluginModules} = plugin_modules(UpdatedConfig),
-
-            %% Finally, process the current working directory
-            ?DEBUG("Command: ~p Modules: ~p Plugins: ~p\n", [Command, Modules, PluginModules]),
-            apply_command(Command, Modules ++ PluginModules, UpdatedConfig, ModuleSetFile)
-    end,
-
-    %% Repeat the process if there are new SeenDirs
-    case NewProcessedDirs =:= ProcessedDirs of
-        true ->
-            ok;
-        false ->
-            process_subdirs(Dir, Modules, UpdatedConfig, ModuleSetFile, Command,
-                            NewProcessedDirs)
+            DirSet2 = process_dir(Dir, Config, Command, DirSet),
+            process_each(Rest, Command, Config, ModuleSetFile, DirSet2)
     end.
 
+
 %%
 %% Given a list of module sets from rebar.app and a directory, find
 %% the appropriate subset of modules for this directory
 %%
 choose_module_set([], _Dir) ->
     {[], undefined};
-choose_module_set([{Fn, Modules} | Rest], Dir) ->
-    case ?MODULE:Fn(Dir) of
+choose_module_set([{Type, Modules} | Rest], Dir) ->
+    case is_dir_type(Type, Dir) of
         {true, File} ->
             {Modules, File};
         false ->
             choose_module_set(Rest, Dir)
     end.
 
-%%
-%% Add ebin to path if there are any local plugin modules for this app.
-%%
-prep_plugin_modules(Config) ->
-    case rebar_config:get_local(Config, rebar_plugins, []) of
-        [_H | _T] ->
-            code:add_path(filename:join([rebar_utils:get_cwd(), "ebin"]));
-        _ ->
-            ok
-    end.
+is_dir_type(app_dir, Dir) ->
+    rebar_app_utils:is_app_dir(Dir);
+is_dir_type(rel_dir, Dir) ->
+    rebar_rel_utils:is_rel_dir(Dir);
+is_dir_type(_, _) ->
+    false.
+
 
 %%
-%% Return a flat list of rebar plugin modules.
+%% Execute a command across all applicable modules
 %%
-plugin_modules(Config) ->
-    Modules = lists:flatten(rebar_config:get_all(Config, rebar_plugins)),
-    plugin_modules(Config, ulist(Modules)).
-
-ulist(L) ->
-    ulist(L, sets:new(), []).
-
-ulist([], _S, Acc) ->
-    lists:reverse(Acc);
-ulist([H | T], S, Acc) ->
-    case sets:is_element(H, S) of
-        true ->
-            ulist(T, S, Acc);
-        false ->
-            ulist(T, sets:add_element(H, S), [H | Acc])
-    end.
-
-plugin_modules(_Config, []) ->
-    {ok, []};
-plugin_modules(_Config, Modules) ->
-    FoundModules = [M || M <- Modules, code:which(M) =/= non_existing],
-    case (Modules =:= FoundModules) of
-        true ->
-            ok;
-        false ->
-            ?DEBUG("Missing plugins: ~p\n", [Modules -- FoundModules]),
-            ok
-    end,
-    {ok, FoundModules}.
-
-%%
-%% Return .app file if the current directory is an OTP app
-%%
-app_dir(Dir) ->
-    rebar_app_utils:is_app_dir(Dir).
-
-%%
-%% Return the reltool.config file if the current directory is release directory
-%%
-rel_dir(Dir) ->
-    rebar_rel_utils:is_rel_dir(Dir).
-
-
-
-
-apply_command(Command, Modules, Config, ModuleFile) ->
+execute(Command, Modules, Config, ModuleFile) ->
     case select_modules(Modules, Command, []) of
         [] ->
             ?WARN("'~p' command does not apply to directory ~s\n",
@@ -441,7 +384,7 @@ apply_command(Command, Modules, Config, ModuleFile) ->
 
 
 update_code_path(Config) ->
-    case rebar_config:get(Config, lib_dirs, []) of
+    case rebar_config:get_local(Config, lib_dirs, []) of
         [] ->
             no_change;
         Paths ->
@@ -490,17 +433,13 @@ run_modules([Module | Rest], Command, Config, File) ->
             {error, Reason}
     end.
 
+
 acc_modules(Modules, Command, Config, File) ->
     acc_modules(select_modules(Modules, Command, []),
                 Command, Config, File, []).
 
-acc_modules([], _Command, Config, _File, Acc) ->
-    {Config, Acc};
+acc_modules([], _Command, _Config, _File, Acc) ->
+    Acc;
 acc_modules([Module | Rest], Command, Config, File, Acc) ->
-    case Module:Command(Config, File) of
-        {ok, NewConfig, Result} when is_list(Result) ->
-            List = Result;
-        {ok, NewConfig, Result} ->
-            List = [Result]
-    end,
-    acc_modules(Rest, Command, NewConfig, File, List ++ Acc).
+    {ok, Dirs} = Module:Command(Config, File),
+    acc_modules(Rest, Command, Config, File, Acc ++ Dirs).

src/rebar_deps.erl

@@ -29,158 +29,128 @@
 -include("rebar.hrl").
 
 -export([preprocess/2,
+         postprocess/2,
          compile/2,
          'check-deps'/2,
-         'get-deps'/2,
-         'delete-deps'/2]).
+         'get-deps'/2]).
+
+
+-record(dep, { dir,
+               app,
+               vsn_regex,
+               source }).
 
 %% ===================================================================
 %% Public API
 %% ===================================================================
 
 preprocess(Config, _) ->
-    DepsDir = get_deps_dir(Config),
-    Config2 = rebar_config:set(Config, deps_dir, DepsDir),
+    %% Get the list of deps for the current working directory and identify those
+    %% deps that are available/present.
+    Deps = rebar_config:get_local(Config, deps, []),
+    {AvailableDeps, MissingDeps} = find_deps(Deps),
 
-    %% Check for available deps, using the list of deps specified in our config.
-    %% We use the directory from the list of tuples for deps with source information to
-    %% update our list of directories to process.
-    case catch(check_deps(rebar_config:get_local(Config, deps, []), [], DepsDir)) of
-        Deps when is_list(Deps) ->
-            %% Walk all the deps and make sure they are available on the code path,
-            %% if the application we're interested in actually exists there.
-            ok = update_deps_code_path(Deps),
-            DepDirs = case rebar_config:get_global(skip_deps, false) of
-                          false ->
-                              [Dir || {Dir, _, _, _} <- Deps];
-                          _Specified ->
-                              []
-                      end,
-            {ok, Config2, DepDirs};
-        {'EXIT', Reason} ->
-            ?ABORT("Error while processing dependencies: ~p\n", [Reason])
+    ?DEBUG("Available deps: ~p\n", [AvailableDeps]),
+    ?DEBUG("Missing deps  : ~p\n", [MissingDeps]),
+
+    %% Add available deps to code path
+    update_deps_code_path(AvailableDeps),
+
+    %% Return all the available dep directories for process
+    %% TODO: Re-add support for skip_deps=true
+    {ok, [D#dep.dir || D <- AvailableDeps]}.
+
+postprocess(_Config, _) ->
+    case erlang:get(?MODULE) of
+        undefined ->
+            {ok, []};
+        Dirs ->
+            erlang:erase(?MODULE),
+            {ok, Dirs}
     end.
 
 compile(Config, AppFile) ->
     'check-deps'(Config, AppFile).
 
 'check-deps'(Config, _) ->
-    %% Get a list of deps that need to be downloaded and display them only
-    DepsDir = get_deps_dir(Config),
-    case catch(check_deps(rebar_config:get_local(Config, deps, []), [], DepsDir)) of
-        [] ->
+    %% Get the list of immediate (i.e. non-transitive) deps that are missing
+    Deps = rebar_config:get_local(Config, deps, []),
+    case find_deps(Deps) of
+        {_, []} ->
+            %% No missing deps
             ok;
-        Deps when is_list(Deps) ->
-            [?CONSOLE("Dependency not available: ~p-~p (~p)\n", [App, VsnRegex, Source]) ||
-                {_Dir, App, VsnRegex, Source} <- Deps],
-            ?FAIL;
-        {'EXIT', Reason} ->
-            ?ABORT("Error while processing dependencies: ~p\n", [Reason])
+        {_, MissingDeps} ->
+            [?CONSOLE("Dependency not available: ~p-~s (~p)\n",
+                      [D#dep.app, D#dep.vsn_regex, D#dep.source]) ||
+                D <- MissingDeps],
+            ?FAIL
     end.
 
 'get-deps'(Config, _) ->
-    DepsDir = get_deps_dir(Config),
+    %% Determine what deps are available and missing
+    Deps = rebar_config:get_local(Config, deps, []),
+    {_AvailableDeps, MissingDeps} = find_deps(Deps),
 
-    %% Get a list of deps that need to be downloaded
-    case catch(check_deps(rebar_config:get_local(Config, deps, []), [], DepsDir)) of
-        Deps when is_list(Deps) ->
-            %% Now for each dependency tuple, pull it
-            [use_source(Dir, App, VsnRegex, Source) || {Dir, App, VsnRegex, Source} <- Deps],
-            ok;
-        {'EXIT', Reason} ->
-            ?ABORT("Error while processing dependencies: ~p\n", [Reason])
-    end.
+    %% For each missing dep with a specified source, try to pull it.
+    PulledDeps = [use_source(D) || D <- MissingDeps, D#dep.source /= undefined],
+
+    %% Add each pulled dep to our list of dirs for post-processing. This yields
+    %% the necessary transitivity of the deps
+    erlang:put(?MODULE, [D#dep.dir || D <- PulledDeps]),
+    ok.
 
-'delete-deps'(Config, _) ->
-    %% Delete all the deps which we downloaded (or would have caused to be
-    %% downloaded).
-    DepsDir = rebar_config:get(Config, deps_dir, rebar_utils:get_cwd()),
-    ?DEBUG("Delete deps: ~p\n", [rebar_config:get(Config, deps, [])]),
-    delete_deps(rebar_config:get_local(Config, deps, []), DepsDir).
 
 %% ===================================================================
 %% Internal functions
 %% ===================================================================
 
-get_deps_dir(Config) ->
-    %% Get the directory where we will place downloaded deps. Take steps
-    %% to ensure that if we're doing a multi-level build, all the deps will
-    %% wind up in a single directory; avoiding potential pain from having
-    %% multiple copies of the same dep scattered throughout the source tree.
-    %%
-    %% The first definition of deps_dir is the one we use; we also fully
-    %% qualify it to ensure everyone sees it properly.
-    case rebar_config:get_all(Config, deps_dir) of
-        [] ->
-            DepsDir = filename:absname("deps");
-        AllDirs ->
-            DepsDir = filename:absname(hd(lists:reverse(AllDirs)))
-    end,
-    ?DEBUG("~s: Using deps dir: ~s\n", [rebar_utils:get_cwd(), DepsDir]),
-    DepsDir.
+get_deps_dir() ->
+    BaseDir = rebar_config:get_global(base_dir, []),
+    filename:join(BaseDir, "deps").
 
 update_deps_code_path([]) ->
     ok;
-update_deps_code_path([{AppDir, App, VsnRegex, _Source} | Rest]) ->
-    case is_app_available(App, VsnRegex, AppDir) of
-        true ->
-            code:add_patha(filename:join(AppDir, ebin));
+update_deps_code_path([Dep | Rest]) ->
+    case is_app_available(Dep#dep.app, Dep#dep.vsn_regex, Dep#dep.dir) of
+        {true, _} ->
+            code:add_patha(filename:join(Dep#dep.dir, ebin));
         false ->
             ok
     end,
     update_deps_code_path(Rest).
 
-check_deps([], Acc, _Dir) ->
-    Acc;
-check_deps([App | Rest], Acc, Dir) when is_atom(App) ->
-    require_app(App, ".*"),
-    check_deps(Rest, Acc, Dir);
-check_deps([{App, VsnRegex} | Rest], Acc, Dir) when is_atom(App) ->
-    require_app(App, VsnRegex),
-    check_deps(Rest, Acc, Dir);
-check_deps([{App, VsnRegex, Source} | Rest], Acc, Dir) ->
+find_deps(Deps) ->
+    find_deps(Deps, {[], []}).
+
+find_deps([], {Avail, Missing}) ->
+    {lists:reverse(Avail), lists:reverse(Missing)};
+find_deps([App | Rest], Acc) when is_atom(App) ->
+    find_deps([{App, ".*", undefined} | Rest], Acc);
+find_deps([{App, VsnRegex} | Rest], Acc) when is_atom(App) ->
+    find_deps([{App, VsnRegex, undefined} | Rest], Acc);
+find_deps([{App, VsnRegex, Source} | Rest], {Avail, Missing}) ->
+    Dep = #dep { app = App,
+                 vsn_regex = VsnRegex,
+                 source = Source },
     case is_app_available(App, VsnRegex) of
-        true ->
-            check_deps(Rest, Acc, Dir);
+        {true, AppDir} ->
+            find_deps(Rest, {[Dep#dep { dir = AppDir } | Avail], Missing});
         false ->
-            %% App is not on our code path OR the version that is available
-            %% doesn't match our regex. Return a tuple containing the target dir
-            %% and source information.
-            AppDir = filename:join(Dir, App),
-            check_deps(Rest, [{AppDir, App, VsnRegex, Source} | Acc], Dir)
+            AppDir = filename:join(get_deps_dir(), Dep#dep.app),
+            case is_app_available(App, VsnRegex, AppDir) of
+                {true, AppDir} ->
+                    find_deps(Rest, {[Dep#dep { dir = AppDir } | Avail], Missing});
+                false ->
+                    find_deps(Rest, {Avail, [Dep#dep { dir = AppDir } | Missing]})
+            end
     end;
-check_deps([Other | _Rest], _Acc, _Dir) ->
+find_deps([Other | _Rest], _Acc) ->
     ?ABORT("Invalid dependency specification ~p in ~s\n",
            [Other, rebar_utils:get_cwd()]).
 
 
-delete_deps([], _DepsDir) ->
-    ok;
-delete_deps([{App, _VsnRegex, _Source} | Rest], DepsDir) ->
-    AppDir = filename:join(DepsDir, App),
-    case filelib:is_dir(AppDir) of
-        true ->
-            ?INFO("Delete dependency dir ~s\n", [AppDir]),
-            rebar_file_utils:rm_rf(AppDir);
-        false ->
-            ok
-    end,
-    delete_deps(Rest, DepsDir);
-delete_deps([_Other | Rest], DepsDir) ->
-    delete_deps(Rest, DepsDir).
 
 
-
-require_app(App, VsnRegex) ->
-    case is_app_available(App, VsnRegex) of
-        true ->
-            ok;
-        false ->
-            %% The requested app is not available on the code path
-            ?ABORT("~s: Dependency ~s-~s not available.\n",
-                   [rebar_utils:get_cwd(), App, VsnRegex])
-    end.
-
 require_source_engine(Source) ->
     case source_engine_avail(Source) of
         true ->
@@ -208,7 +178,7 @@ is_app_available(App, VsnRegex, Path) ->
                           [App, VsnRegex, App, Vsn, Path]),
                     case re:run(Vsn, VsnRegex, [{capture, none}]) of
                         match ->
-                            true;
+                            {true, Path};
                         nomatch ->
                             ?WARN("~s has version ~p; requested regex was ~s\n",
                                   [AppFile, Vsn, VsnRegex]),
@@ -224,32 +194,33 @@ is_app_available(App, VsnRegex, Path) ->
             false
     end.
 
-use_source(AppDir, App, VsnRegex, Source) ->
-    ?CONSOLE("Pulling ~p from ~p\n", [App, Source]),
-    use_source(AppDir, App, VsnRegex, Source, 3).
+use_source(Dep) ->
+    use_source(Dep, 3).
 
-use_source(_AppDir, _App, _VsnRegex, Source, 0) ->
-    ?ABORT("Failed to acquire source from ~p after 3 tries.\n", [Source]);
-use_source(AppDir, App, VsnRegex, Source, Count) ->
-    case filelib:is_dir(AppDir) of
+use_source(Dep, 0) ->
+    ?ABORT("Failed to acquire source from ~p after 3 tries.\n", [Dep#dep.source]);
+use_source(Dep, Count) ->
+    case filelib:is_dir(Dep#dep.dir) of
         true ->
             %% Already downloaded -- verify the versioning matches up with our regex
-            case is_app_available(App, VsnRegex, AppDir) of
-                true ->
+            case is_app_available(Dep#dep.app, Dep#dep.vsn_regex, Dep#dep.dir) of
+                {true, _} ->
                     %% Available version matches up -- we're good to go; add the
                     %% app dir to our code path
-                    code:add_patha(filename:join(AppDir, ebin)),
-                    ok;
+                    code:add_patha(filename:join(Dep#dep.dir, ebin)),
+                    Dep;
                 false ->
                     %% The app that was downloaded doesn't match up (or had
                     %% errors or something). For the time being, abort.
                     ?ABORT("Dependency dir ~s does not satisfy version regex ~s.\n",
-                           [AppDir, VsnRegex])
+                           [Dep#dep.dir, Dep#dep.vsn_regex])
             end;
         false ->
-            require_source_engine(Source),
-            download_source(AppDir, Source),
-            use_source(AppDir, App, VsnRegex, Source, Count-1)
+            ?CONSOLE("Pulling ~p from ~p\n", [Dep#dep.app, Dep#dep.source]),
+            require_source_engine(Dep#dep.source),
+            TargetDir = filename:join(get_deps_dir(), Dep#dep.app),
+            download_source(TargetDir, Dep#dep.source),
+            use_source(Dep#dep { dir = TargetDir }, Count-1)
     end.
 
 download_source(AppDir, {hg, Url, Rev}) ->

src/rebar_subdirs.erl

@@ -38,5 +38,5 @@ preprocess(Config, _) ->
     %% Get the list of subdirs specified in the config (if any).
     Cwd = rebar_utils:get_cwd(),
     Subdirs = [filename:join(Cwd, Dir) || Dir <- rebar_config:get_local(Config, sub_dirs, [])],
-    {ok, Config, Subdirs}.
+    {ok, Subdirs}.