From 3df1d4292aa4efd6fbe1f154f9593b31ffff0c0a Mon Sep 17 00:00:00 2001 From: Dave Smith Date: Wed, 9 Jun 2010 13:16:58 -0600 Subject: [PATCH] Heavy-duty refactor to support truly transitive dependencies --HG-- extra : rebase_source : 41c7f1c337a7cb63582aecd7b87ba998b40ba3aa --- src/rebar_core.erl | 195 +++++++++++++------------------------ src/rebar_deps.erl | 221 ++++++++++++++++++------------------------ src/rebar_subdirs.erl | 2 +- 3 files changed, 164 insertions(+), 254 deletions(-) diff --git a/src/rebar_core.erl b/src/rebar_core.erl index 6cf7bff..8177c43 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -28,8 +28,6 @@ -export([run/1]). --export([app_dir/1, rel_dir/1]). % Ugh - -include("rebar.hrl"). @@ -71,8 +69,11 @@ run(RawArgs) -> rebar_config:set_global(escript, filename:absname(escript:script_name())), ?DEBUG("Rebar location: ~p\n", [rebar_config:get_global(escript, undefined)]), + %% Note the top-level directory for reference + rebar_config:set_global(base_dir, filename:absname(rebar_utils:get_cwd())), + %% Load rebar.config, if it exists - [process_dir(rebar_utils:get_cwd(), rebar_config:new(), Command) + [process_dir(rebar_utils:get_cwd(), rebar_config:new(), Command, sets:new()) || Command <- CommandAtoms], ok. @@ -250,13 +251,14 @@ filter_flags([Item | Rest], Commands) -> end. -process_dir(Dir, ParentConfig, Command) -> +process_dir(Dir, ParentConfig, Command, DirSet) -> case filelib:is_dir(Dir) of false -> ?WARN("Skipping non-existent sub-dir: ~p\n", [Dir]), - ok; + DirSet; true -> + ?DEBUG("Entering ~s\n", [Dir]), ok = file:set_cwd(Dir), Config = rebar_config:new(ParentConfig), @@ -272,151 +274,92 @@ process_dir(Dir, ParentConfig, Command) -> {ok, AvailModuleSets} = application:get_env(rebar, modules), {DirModules, ModuleSetFile} = choose_module_set(AvailModuleSets, Dir), - %% Get the list of modules for "any dir". This is a catch-all list of modules - %% that are processed in addition to modules associated with this directory - %% type. These any_dir modules are processed FIRST. + %% Get the list of modules for "any dir". This is a catch-all list + %% of modules that are processed in addition to modules associated + %% with this directory type. These any_dir modules are processed + %% FIRST. {ok, AnyDirModules} = application:get_env(rebar, any_dir_modules), Modules = AnyDirModules ++ DirModules, - ok = process_subdirs(Dir, Modules, Config, ModuleSetFile, Command), + %% Invoke 'preprocess' on the modules -- this yields a list of other + %% directories that should be processed _before_ the current one. + Predirs = acc_modules(Modules, preprocess, Config, ModuleSetFile), + ?DEBUG("Predirs: ~p\n", [Predirs]), + DirSet2 = process_each(Predirs, Command, Config, ModuleSetFile, DirSet), + + %% Make sure the CWD is reset properly; processing the dirs may have + %% caused it to change + ok = file:set_cwd(Dir), + + %% Execute the current command on this directory + execute(Command, Modules, Config, ModuleSetFile), + + %% Mark the current directory as processed + DirSet3 = sets:add_element(Dir, DirSet2), + + %% Invoke 'postprocess' on the modules -- this yields a list of other + %% directories that should be processed _after_ the current one. + Postdirs = acc_modules(Modules, postprocess, Config, ModuleSetFile), + ?DEBUG("Postdirs: ~p\n", [Postdirs]), + DirSet4 = process_each(Postdirs, Command, Config, ModuleSetFile, DirSet3), + + %% Make sure the CWD is reset properly; processing the dirs may have + %% caused it to change + ok = file:set_cwd(Dir), %% Once we're all done processing, reset the code path to whatever %% the parent initialized it to restore_code_path(CurrentCodePath), - ok + + %% Return the updated dirset as our result + DirSet4 end. %% -%% Run the preprocessors and execute the command on all newly -%% found Dirs until no new Dirs are found by the preprocessors. +%% Given a list of directories and a set of previously processed directories, +%% process each one we haven't seen yet %% -process_subdirs(Dir, Modules, Config, ModuleSetFile, Command) -> - process_subdirs(Dir, Modules, Config, ModuleSetFile, Command, sets:new()). - -process_subdirs(Dir, Modules, Config, ModuleSetFile, Command, ProcessedDirs) -> - %% Give the modules a chance to tweak config and indicate if there - %% are any other dirs that might need processing first. - {UpdatedConfig, Dirs} = acc_modules(Modules, preprocess, Config, ModuleSetFile), - ?DEBUG("~s subdirs: ~p\n", [Dir, Dirs]), - - %% Add ebin to path if this app has any plugins configured locally. - prep_plugin_modules(UpdatedConfig), - - %% Process subdirs that haven't already been processed. - F = fun (D, S) -> - case filelib:is_dir(D) andalso (not sets:is_element(D, S)) of - true -> - process_dir(D, UpdatedConfig, Command), - sets:add_element(D, S); - false -> - S - end - end, - NewProcessedDirs = lists:foldl(F, sets:add_element(parent, ProcessedDirs), Dirs), - - %% Make sure the CWD is reset properly; processing subdirs may have caused it - %% to change - ok = file:set_cwd(Dir), - - %% Run the parent commands exactly once as well - case sets:is_element(parent, ProcessedDirs) of +process_each([], _Command, _Config, _ModuleSetFile, DirSet) -> + DirSet; +process_each([Dir | Rest], Command, Config, ModuleSetFile, DirSet) -> + case sets:is_element(Dir, DirSet) of true -> - ok; + ?DEBUG("Skipping ~s; already processed!\n", [Dir]), + process_each(Rest, Command, Config, ModuleSetFile, DirSet); false -> - %% Get the list of plug-in modules from rebar.config. These modules are - %% processed LAST and do not participate in preprocess. - {ok, PluginModules} = plugin_modules(UpdatedConfig), - - %% Finally, process the current working directory - ?DEBUG("Command: ~p Modules: ~p Plugins: ~p\n", [Command, Modules, PluginModules]), - apply_command(Command, Modules ++ PluginModules, UpdatedConfig, ModuleSetFile) - end, - - %% Repeat the process if there are new SeenDirs - case NewProcessedDirs =:= ProcessedDirs of - true -> - ok; - false -> - process_subdirs(Dir, Modules, UpdatedConfig, ModuleSetFile, Command, - NewProcessedDirs) + DirSet2 = process_dir(Dir, Config, Command, DirSet), + process_each(Rest, Command, Config, ModuleSetFile, DirSet2) end. + %% %% Given a list of module sets from rebar.app and a directory, find %% the appropriate subset of modules for this directory %% choose_module_set([], _Dir) -> {[], undefined}; -choose_module_set([{Fn, Modules} | Rest], Dir) -> - case ?MODULE:Fn(Dir) of +choose_module_set([{Type, Modules} | Rest], Dir) -> + case is_dir_type(Type, Dir) of {true, File} -> {Modules, File}; false -> choose_module_set(Rest, Dir) end. -%% -%% Add ebin to path if there are any local plugin modules for this app. -%% -prep_plugin_modules(Config) -> - case rebar_config:get_local(Config, rebar_plugins, []) of - [_H | _T] -> - code:add_path(filename:join([rebar_utils:get_cwd(), "ebin"])); - _ -> - ok - end. +is_dir_type(app_dir, Dir) -> + rebar_app_utils:is_app_dir(Dir); +is_dir_type(rel_dir, Dir) -> + rebar_rel_utils:is_rel_dir(Dir); +is_dir_type(_, _) -> + false. + %% -%% Return a flat list of rebar plugin modules. +%% Execute a command across all applicable modules %% -plugin_modules(Config) -> - Modules = lists:flatten(rebar_config:get_all(Config, rebar_plugins)), - plugin_modules(Config, ulist(Modules)). - -ulist(L) -> - ulist(L, sets:new(), []). - -ulist([], _S, Acc) -> - lists:reverse(Acc); -ulist([H | T], S, Acc) -> - case sets:is_element(H, S) of - true -> - ulist(T, S, Acc); - false -> - ulist(T, sets:add_element(H, S), [H | Acc]) - end. - -plugin_modules(_Config, []) -> - {ok, []}; -plugin_modules(_Config, Modules) -> - FoundModules = [M || M <- Modules, code:which(M) =/= non_existing], - case (Modules =:= FoundModules) of - true -> - ok; - false -> - ?DEBUG("Missing plugins: ~p\n", [Modules -- FoundModules]), - ok - end, - {ok, FoundModules}. - -%% -%% Return .app file if the current directory is an OTP app -%% -app_dir(Dir) -> - rebar_app_utils:is_app_dir(Dir). - -%% -%% Return the reltool.config file if the current directory is release directory -%% -rel_dir(Dir) -> - rebar_rel_utils:is_rel_dir(Dir). - - - - -apply_command(Command, Modules, Config, ModuleFile) -> +execute(Command, Modules, Config, ModuleFile) -> case select_modules(Modules, Command, []) of [] -> ?WARN("'~p' command does not apply to directory ~s\n", @@ -441,7 +384,7 @@ apply_command(Command, Modules, Config, ModuleFile) -> update_code_path(Config) -> - case rebar_config:get(Config, lib_dirs, []) of + case rebar_config:get_local(Config, lib_dirs, []) of [] -> no_change; Paths -> @@ -490,17 +433,13 @@ run_modules([Module | Rest], Command, Config, File) -> {error, Reason} end. + acc_modules(Modules, Command, Config, File) -> acc_modules(select_modules(Modules, Command, []), Command, Config, File, []). -acc_modules([], _Command, Config, _File, Acc) -> - {Config, Acc}; +acc_modules([], _Command, _Config, _File, Acc) -> + Acc; acc_modules([Module | Rest], Command, Config, File, Acc) -> - case Module:Command(Config, File) of - {ok, NewConfig, Result} when is_list(Result) -> - List = Result; - {ok, NewConfig, Result} -> - List = [Result] - end, - acc_modules(Rest, Command, NewConfig, File, List ++ Acc). + {ok, Dirs} = Module:Command(Config, File), + acc_modules(Rest, Command, Config, File, Acc ++ Dirs). diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index 7d07ff4..3b57461 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -29,158 +29,128 @@ -include("rebar.hrl"). -export([preprocess/2, + postprocess/2, compile/2, 'check-deps'/2, - 'get-deps'/2, - 'delete-deps'/2]). + 'get-deps'/2]). + + +-record(dep, { dir, + app, + vsn_regex, + source }). %% =================================================================== %% Public API %% =================================================================== preprocess(Config, _) -> - DepsDir = get_deps_dir(Config), - Config2 = rebar_config:set(Config, deps_dir, DepsDir), + %% Get the list of deps for the current working directory and identify those + %% deps that are available/present. + Deps = rebar_config:get_local(Config, deps, []), + {AvailableDeps, MissingDeps} = find_deps(Deps), - %% Check for available deps, using the list of deps specified in our config. - %% We use the directory from the list of tuples for deps with source information to - %% update our list of directories to process. - case catch(check_deps(rebar_config:get_local(Config, deps, []), [], DepsDir)) of - Deps when is_list(Deps) -> - %% Walk all the deps and make sure they are available on the code path, - %% if the application we're interested in actually exists there. - ok = update_deps_code_path(Deps), - DepDirs = case rebar_config:get_global(skip_deps, false) of - false -> - [Dir || {Dir, _, _, _} <- Deps]; - _Specified -> - [] - end, - {ok, Config2, DepDirs}; - {'EXIT', Reason} -> - ?ABORT("Error while processing dependencies: ~p\n", [Reason]) + ?DEBUG("Available deps: ~p\n", [AvailableDeps]), + ?DEBUG("Missing deps : ~p\n", [MissingDeps]), + + %% Add available deps to code path + update_deps_code_path(AvailableDeps), + + %% Return all the available dep directories for process + %% TODO: Re-add support for skip_deps=true + {ok, [D#dep.dir || D <- AvailableDeps]}. + +postprocess(_Config, _) -> + case erlang:get(?MODULE) of + undefined -> + {ok, []}; + Dirs -> + erlang:erase(?MODULE), + {ok, Dirs} end. compile(Config, AppFile) -> 'check-deps'(Config, AppFile). 'check-deps'(Config, _) -> - %% Get a list of deps that need to be downloaded and display them only - DepsDir = get_deps_dir(Config), - case catch(check_deps(rebar_config:get_local(Config, deps, []), [], DepsDir)) of - [] -> + %% Get the list of immediate (i.e. non-transitive) deps that are missing + Deps = rebar_config:get_local(Config, deps, []), + case find_deps(Deps) of + {_, []} -> + %% No missing deps ok; - Deps when is_list(Deps) -> - [?CONSOLE("Dependency not available: ~p-~p (~p)\n", [App, VsnRegex, Source]) || - {_Dir, App, VsnRegex, Source} <- Deps], - ?FAIL; - {'EXIT', Reason} -> - ?ABORT("Error while processing dependencies: ~p\n", [Reason]) + {_, MissingDeps} -> + [?CONSOLE("Dependency not available: ~p-~s (~p)\n", + [D#dep.app, D#dep.vsn_regex, D#dep.source]) || + D <- MissingDeps], + ?FAIL end. 'get-deps'(Config, _) -> - DepsDir = get_deps_dir(Config), + %% Determine what deps are available and missing + Deps = rebar_config:get_local(Config, deps, []), + {_AvailableDeps, MissingDeps} = find_deps(Deps), - %% Get a list of deps that need to be downloaded - case catch(check_deps(rebar_config:get_local(Config, deps, []), [], DepsDir)) of - Deps when is_list(Deps) -> - %% Now for each dependency tuple, pull it - [use_source(Dir, App, VsnRegex, Source) || {Dir, App, VsnRegex, Source} <- Deps], - ok; - {'EXIT', Reason} -> - ?ABORT("Error while processing dependencies: ~p\n", [Reason]) - end. + %% For each missing dep with a specified source, try to pull it. + PulledDeps = [use_source(D) || D <- MissingDeps, D#dep.source /= undefined], + + %% Add each pulled dep to our list of dirs for post-processing. This yields + %% the necessary transitivity of the deps + erlang:put(?MODULE, [D#dep.dir || D <- PulledDeps]), + ok. -'delete-deps'(Config, _) -> - %% Delete all the deps which we downloaded (or would have caused to be - %% downloaded). - DepsDir = rebar_config:get(Config, deps_dir, rebar_utils:get_cwd()), - ?DEBUG("Delete deps: ~p\n", [rebar_config:get(Config, deps, [])]), - delete_deps(rebar_config:get_local(Config, deps, []), DepsDir). %% =================================================================== %% Internal functions %% =================================================================== -get_deps_dir(Config) -> - %% Get the directory where we will place downloaded deps. Take steps - %% to ensure that if we're doing a multi-level build, all the deps will - %% wind up in a single directory; avoiding potential pain from having - %% multiple copies of the same dep scattered throughout the source tree. - %% - %% The first definition of deps_dir is the one we use; we also fully - %% qualify it to ensure everyone sees it properly. - case rebar_config:get_all(Config, deps_dir) of - [] -> - DepsDir = filename:absname("deps"); - AllDirs -> - DepsDir = filename:absname(hd(lists:reverse(AllDirs))) - end, - ?DEBUG("~s: Using deps dir: ~s\n", [rebar_utils:get_cwd(), DepsDir]), - DepsDir. +get_deps_dir() -> + BaseDir = rebar_config:get_global(base_dir, []), + filename:join(BaseDir, "deps"). update_deps_code_path([]) -> ok; -update_deps_code_path([{AppDir, App, VsnRegex, _Source} | Rest]) -> - case is_app_available(App, VsnRegex, AppDir) of - true -> - code:add_patha(filename:join(AppDir, ebin)); +update_deps_code_path([Dep | Rest]) -> + case is_app_available(Dep#dep.app, Dep#dep.vsn_regex, Dep#dep.dir) of + {true, _} -> + code:add_patha(filename:join(Dep#dep.dir, ebin)); false -> ok end, update_deps_code_path(Rest). -check_deps([], Acc, _Dir) -> - Acc; -check_deps([App | Rest], Acc, Dir) when is_atom(App) -> - require_app(App, ".*"), - check_deps(Rest, Acc, Dir); -check_deps([{App, VsnRegex} | Rest], Acc, Dir) when is_atom(App) -> - require_app(App, VsnRegex), - check_deps(Rest, Acc, Dir); -check_deps([{App, VsnRegex, Source} | Rest], Acc, Dir) -> +find_deps(Deps) -> + find_deps(Deps, {[], []}). + +find_deps([], {Avail, Missing}) -> + {lists:reverse(Avail), lists:reverse(Missing)}; +find_deps([App | Rest], Acc) when is_atom(App) -> + find_deps([{App, ".*", undefined} | Rest], Acc); +find_deps([{App, VsnRegex} | Rest], Acc) when is_atom(App) -> + find_deps([{App, VsnRegex, undefined} | Rest], Acc); +find_deps([{App, VsnRegex, Source} | Rest], {Avail, Missing}) -> + Dep = #dep { app = App, + vsn_regex = VsnRegex, + source = Source }, case is_app_available(App, VsnRegex) of - true -> - check_deps(Rest, Acc, Dir); + {true, AppDir} -> + find_deps(Rest, {[Dep#dep { dir = AppDir } | Avail], Missing}); false -> - %% App is not on our code path OR the version that is available - %% doesn't match our regex. Return a tuple containing the target dir - %% and source information. - AppDir = filename:join(Dir, App), - check_deps(Rest, [{AppDir, App, VsnRegex, Source} | Acc], Dir) + AppDir = filename:join(get_deps_dir(), Dep#dep.app), + case is_app_available(App, VsnRegex, AppDir) of + {true, AppDir} -> + find_deps(Rest, {[Dep#dep { dir = AppDir } | Avail], Missing}); + false -> + find_deps(Rest, {Avail, [Dep#dep { dir = AppDir } | Missing]}) + end end; -check_deps([Other | _Rest], _Acc, _Dir) -> +find_deps([Other | _Rest], _Acc) -> ?ABORT("Invalid dependency specification ~p in ~s\n", [Other, rebar_utils:get_cwd()]). -delete_deps([], _DepsDir) -> - ok; -delete_deps([{App, _VsnRegex, _Source} | Rest], DepsDir) -> - AppDir = filename:join(DepsDir, App), - case filelib:is_dir(AppDir) of - true -> - ?INFO("Delete dependency dir ~s\n", [AppDir]), - rebar_file_utils:rm_rf(AppDir); - false -> - ok - end, - delete_deps(Rest, DepsDir); -delete_deps([_Other | Rest], DepsDir) -> - delete_deps(Rest, DepsDir). - -require_app(App, VsnRegex) -> - case is_app_available(App, VsnRegex) of - true -> - ok; - false -> - %% The requested app is not available on the code path - ?ABORT("~s: Dependency ~s-~s not available.\n", - [rebar_utils:get_cwd(), App, VsnRegex]) - end. - require_source_engine(Source) -> case source_engine_avail(Source) of true -> @@ -208,7 +178,7 @@ is_app_available(App, VsnRegex, Path) -> [App, VsnRegex, App, Vsn, Path]), case re:run(Vsn, VsnRegex, [{capture, none}]) of match -> - true; + {true, Path}; nomatch -> ?WARN("~s has version ~p; requested regex was ~s\n", [AppFile, Vsn, VsnRegex]), @@ -224,32 +194,33 @@ is_app_available(App, VsnRegex, Path) -> false end. -use_source(AppDir, App, VsnRegex, Source) -> - ?CONSOLE("Pulling ~p from ~p\n", [App, Source]), - use_source(AppDir, App, VsnRegex, Source, 3). +use_source(Dep) -> + use_source(Dep, 3). -use_source(_AppDir, _App, _VsnRegex, Source, 0) -> - ?ABORT("Failed to acquire source from ~p after 3 tries.\n", [Source]); -use_source(AppDir, App, VsnRegex, Source, Count) -> - case filelib:is_dir(AppDir) of +use_source(Dep, 0) -> + ?ABORT("Failed to acquire source from ~p after 3 tries.\n", [Dep#dep.source]); +use_source(Dep, Count) -> + case filelib:is_dir(Dep#dep.dir) of true -> %% Already downloaded -- verify the versioning matches up with our regex - case is_app_available(App, VsnRegex, AppDir) of - true -> + case is_app_available(Dep#dep.app, Dep#dep.vsn_regex, Dep#dep.dir) of + {true, _} -> %% Available version matches up -- we're good to go; add the %% app dir to our code path - code:add_patha(filename:join(AppDir, ebin)), - ok; + code:add_patha(filename:join(Dep#dep.dir, ebin)), + Dep; false -> %% The app that was downloaded doesn't match up (or had %% errors or something). For the time being, abort. ?ABORT("Dependency dir ~s does not satisfy version regex ~s.\n", - [AppDir, VsnRegex]) + [Dep#dep.dir, Dep#dep.vsn_regex]) end; false -> - require_source_engine(Source), - download_source(AppDir, Source), - use_source(AppDir, App, VsnRegex, Source, Count-1) + ?CONSOLE("Pulling ~p from ~p\n", [Dep#dep.app, Dep#dep.source]), + require_source_engine(Dep#dep.source), + TargetDir = filename:join(get_deps_dir(), Dep#dep.app), + download_source(TargetDir, Dep#dep.source), + use_source(Dep#dep { dir = TargetDir }, Count-1) end. download_source(AppDir, {hg, Url, Rev}) -> diff --git a/src/rebar_subdirs.erl b/src/rebar_subdirs.erl index 81f588d..7e39ffa 100644 --- a/src/rebar_subdirs.erl +++ b/src/rebar_subdirs.erl @@ -38,5 +38,5 @@ preprocess(Config, _) -> %% Get the list of subdirs specified in the config (if any). Cwd = rebar_utils:get_cwd(), Subdirs = [filename:join(Cwd, Dir) || Dir <- rebar_config:get_local(Config, sub_dirs, [])], - {ok, Config, Subdirs}. + {ok, Subdirs}.