diff --git a/inttest/tdeps_update/a.erl b/inttest/tdeps_update/a.erl new file mode 100644 index 0000000..294ae21 --- /dev/null +++ b/inttest/tdeps_update/a.erl @@ -0,0 +1,3 @@ +-module({{module}}). + +-include_lib("b/include/b.hrl"). diff --git a/inttest/tdeps_update/a.rebar.config b/inttest/tdeps_update/a.rebar.config new file mode 100644 index 0000000..3b721dc --- /dev/null +++ b/inttest/tdeps_update/a.rebar.config @@ -0,0 +1 @@ +{deps, [{b, "0.2.3", {git, "../repo/b", {tag, "0.2.3"}}}]}. diff --git a/inttest/tdeps_update/a2.rebar.config b/inttest/tdeps_update/a2.rebar.config new file mode 100644 index 0000000..5687349 --- /dev/null +++ b/inttest/tdeps_update/a2.rebar.config @@ -0,0 +1 @@ +{deps, [{b, "0.2.4", {git, "../repo/b", {tag, "0.2.4"}}}]}. diff --git a/inttest/tdeps_update/a3.rebar.config b/inttest/tdeps_update/a3.rebar.config new file mode 100644 index 0000000..86bf462 --- /dev/null +++ b/inttest/tdeps_update/a3.rebar.config @@ -0,0 +1 @@ +{deps, [{b, "0.2.5", {git, "../repo/b", {tag, "0.2.5"}}}]}. diff --git a/inttest/tdeps_update/a4.rebar.config b/inttest/tdeps_update/a4.rebar.config new file mode 100644 index 0000000..bfba813 --- /dev/null +++ b/inttest/tdeps_update/a4.rebar.config @@ -0,0 +1,4 @@ +{deps, [ + {b, "0.2.6", {git, "../repo/b", {tag, "0.2.6"}}}, + {f, "0.1", {git, "../repo/f", {tag, "0.1"}}} + ]}. diff --git a/inttest/tdeps_update/b.hrl b/inttest/tdeps_update/b.hrl new file mode 100644 index 0000000..efbeab1 --- /dev/null +++ b/inttest/tdeps_update/b.hrl @@ -0,0 +1 @@ +-include_lib("c/include/c.hrl"). diff --git a/inttest/tdeps_update/b.rebar.config b/inttest/tdeps_update/b.rebar.config new file mode 100644 index 0000000..536aaa9 --- /dev/null +++ b/inttest/tdeps_update/b.rebar.config @@ -0,0 +1 @@ +{deps, [{c, "1.0", {git, "../repo/c", {tag, "1.0"}}}]}. diff --git a/inttest/tdeps_update/b2.rebar.config b/inttest/tdeps_update/b2.rebar.config new file mode 100644 index 0000000..b603277 --- /dev/null +++ b/inttest/tdeps_update/b2.rebar.config @@ -0,0 +1 @@ +{deps, [{c, "1.1", {git, "../repo/c", {tag, "1.1"}}}]}. diff --git a/inttest/tdeps_update/b3.rebar.config b/inttest/tdeps_update/b3.rebar.config new file mode 100644 index 0000000..5f4e20a --- /dev/null +++ b/inttest/tdeps_update/b3.rebar.config @@ -0,0 +1 @@ +{deps, [{c, "1.2", {git, "../repo/c", {tag, "1.2"}}}]}. diff --git a/inttest/tdeps_update/b4.rebar.config b/inttest/tdeps_update/b4.rebar.config new file mode 100644 index 0000000..5fd1dca --- /dev/null +++ b/inttest/tdeps_update/b4.rebar.config @@ -0,0 +1 @@ +{deps, [{c, "1.3", {git, "../repo/c", {tag, "1.3"}}}]}. diff --git a/inttest/tdeps_update/c.hrl b/inttest/tdeps_update/c.hrl new file mode 100644 index 0000000..9f02fab --- /dev/null +++ b/inttest/tdeps_update/c.hrl @@ -0,0 +1 @@ +-define(HELLO, hello). diff --git a/inttest/tdeps_update/c.rebar.config b/inttest/tdeps_update/c.rebar.config new file mode 100644 index 0000000..d99b963 --- /dev/null +++ b/inttest/tdeps_update/c.rebar.config @@ -0,0 +1 @@ +{deps, [{d, "0.7", {git, "../repo/d", {tag, "0.7"}}}]}. diff --git a/inttest/tdeps_update/c2.hrl b/inttest/tdeps_update/c2.hrl new file mode 100644 index 0000000..cc87fff --- /dev/null +++ b/inttest/tdeps_update/c2.hrl @@ -0,0 +1 @@ +-include_lib("d/include/d.hrl"). diff --git a/inttest/tdeps_update/c2.rebar.config b/inttest/tdeps_update/c2.rebar.config new file mode 100644 index 0000000..1297e07 --- /dev/null +++ b/inttest/tdeps_update/c2.rebar.config @@ -0,0 +1,4 @@ +{deps, [ + {d, "0.7", {git, "../repo/d", {tag, "0.7"}}}, + {e, "2.0", {git, "../repo/e", {tag, "2.0"}}} + ]}. diff --git a/inttest/tdeps_update/c3.rebar.config b/inttest/tdeps_update/c3.rebar.config new file mode 100644 index 0000000..40c93c5 --- /dev/null +++ b/inttest/tdeps_update/c3.rebar.config @@ -0,0 +1,4 @@ +{deps, [ + {d, "0.7", {git, "../repo/d", {tag, "0.7"}}}, + {e, "2.1", {git, "../repo/e", {tag, "2.1"}}} + ]}. diff --git a/inttest/tdeps_update/d.hrl b/inttest/tdeps_update/d.hrl new file mode 100644 index 0000000..9f02fab --- /dev/null +++ b/inttest/tdeps_update/d.hrl @@ -0,0 +1 @@ +-define(HELLO, hello). diff --git a/inttest/tdeps_update/root.rebar.config b/inttest/tdeps_update/root.rebar.config new file mode 100644 index 0000000..ea03437 --- /dev/null +++ b/inttest/tdeps_update/root.rebar.config @@ -0,0 +1 @@ +{sub_dirs, ["apps/a1"]}. diff --git a/inttest/tdeps_update/tdeps_update_rt.erl b/inttest/tdeps_update/tdeps_update_rt.erl new file mode 100644 index 0000000..81bb7ef --- /dev/null +++ b/inttest/tdeps_update/tdeps_update_rt.erl @@ -0,0 +1,147 @@ +-module(tdeps_update_rt). + +-compile(export_all). + +%% Exercises update deps, with recursive dependency updates. +%% Initially: +%% A(v0.5) -> B(v0.2.3) -> C(v1.0) +%% But after updating A to 0.6: +%% A(v0.6) -> B(v0.2.4) -> C(v1.1) +%% -> D(v0.7) +%% And after updating A to 0.7: +%% A(v0.7) -> B(v0.2.5) -> C(v1.2) -> E(v2.0) +%% -> D(v0.7) +%% And after updating A to 0.8: +%% A(v0.8) -> B(v0.2.6) -> C(v1.3) -> E(v2.1) +%% -> D(v0.7) +%% -> F(v0.1) -> E(v2.1) +files() -> + [ + %% A1 application + {create, "apps/a1/ebin/a1.app", app(a1, [a1], "0.5")}, + {copy, "a.rebar.config", "apps/a1/rebar.config"}, + {template, "a.erl", "apps/a1/src/a1.erl", dict:from_list([{module, a1}])}, + + {copy, "root.rebar.config", "rebar.config"}, + {copy, "../../rebar", "rebar"}, + + %% B application + {create, "repo/b/ebin/b.app", app(b, [], "0.2.3")}, + {create, "b2.app", app(b, [], "0.2.4")}, + {create, "b3.app", app(b, [], "0.2.5")}, + {create, "b4.app", app(b, [], "0.2.6")}, + {copy, "b.rebar.config", "repo/b/rebar.config"}, + {copy, "b.hrl", "repo/b/include/b.hrl"}, + + %% C application + {create, "repo/c/ebin/c.app", app(c, [], "1.0")}, + {create, "c2.app", app(c, [], "1.1")}, + {create, "c3.app", app(c, [], "1.2")}, + {create, "c4.app", app(c, [], "1.3")}, + {copy, "c.hrl", "repo/c/include/c.hrl"}, + + %% D application + {create, "repo/d/ebin/d.app", app(d, [], "0.7")}, + {copy, "d.hrl", "repo/d/include/d.hrl"}, + + %% E application + {create, "repo/e/ebin/e.app", app(e, [], "2.0")}, + {create, "e2.app", app(e, [], "2.1")}, + + %% F application + {create, "repo/f/ebin/f.app", app(f, [], "0.1")}, + + %% update files + {copy, "a2.rebar.config", "a2.rebar.config"}, + {copy, "a3.rebar.config", "a3.rebar.config"}, + {copy, "a4.rebar.config", "a4.rebar.config"}, + {copy, "b2.rebar.config", "b2.rebar.config"}, + {copy, "b3.rebar.config", "b3.rebar.config"}, + {copy, "b4.rebar.config", "b4.rebar.config"}, + {copy, "c2.hrl", "c2.hrl"}, + {copy, "c.rebar.config", "c.rebar.config"}, + {copy, "c2.rebar.config", "c2.rebar.config"}, + {copy, "c3.rebar.config", "c3.rebar.config"} + ]. + +apply_cmds([], _Params) -> + ok; +apply_cmds([Cmd | Rest], Params) -> + io:format("Running: ~s (~p)\n", [Cmd, Params]), + {ok, _} = retest_sh:run(Cmd, Params), + apply_cmds(Rest, Params). + +run(_Dir) -> + %% Initialize the b/c/d apps as git repos so that dependencies pull + %% properly + GitCmds = ["git init", + "git add -A", + "git config user.email 'tdeps@example.com'", + "git config user.name 'tdeps'", + "git commit -a -m 'Initial Commit'"], + BCmds = ["git tag 0.2.3", + "cp ../../b2.rebar.config rebar.config", + "cp ../../b2.app ebin/b.app", + "git commit -a -m 'update to 0.2.4'", + "git tag 0.2.4", + "cp ../../b3.rebar.config rebar.config", + "cp ../../b3.app ebin/b.app", + "git commit -a -m 'update to 0.2.5'", + "git tag 0.2.5", + "cp ../../b4.rebar.config rebar.config", + "cp ../../b4.app ebin/b.app", + "git commit -a -m 'update to 0.2.6'", + "git tag 0.2.6"], + %"git checkout 0.2.3"], + CCmds = ["git tag 1.0", + "cp ../../c2.hrl include/c.hrl", + "cp ../../c2.app ebin/c.app", + "cp ../../c.rebar.config rebar.config", + "git add rebar.config", + "git commit -a -m 'update to 1.1'", + "git tag 1.1", + "cp ../../c3.app ebin/c.app", + "cp ../../c2.rebar.config rebar.config", + "git commit -a -m 'update to 1.2'", + "git tag 1.2", + "cp ../../c4.app ebin/c.app", + "cp ../../c3.rebar.config rebar.config", + "git commit -a -m 'update to 1.3'", + "git tag 1.3"], + %"git checkout 1.0"], + DCmds = ["git tag 0.7"], + ECmds = ["git tag 2.0", + "cp ../../e2.app ebin/e.app", + "git commit -a -m 'update to 2.1'", + "git tag 2.1"], + FCmds = ["git tag 0.1"], + + ok = apply_cmds(GitCmds++BCmds, [{dir, "repo/b"}]), + ok = apply_cmds(GitCmds++CCmds, [{dir, "repo/c"}]), + ok = apply_cmds(GitCmds++DCmds, [{dir, "repo/d"}]), + ok = apply_cmds(GitCmds++ECmds, [{dir, "repo/e"}]), + ok = apply_cmds(GitCmds++FCmds, [{dir, "repo/f"}]), + + {ok, _} = retest_sh:run("./rebar -v get-deps compile", []), + os:cmd("cp a2.rebar.config apps/a1/rebar.config"), + {ok, _} = retest_sh:run("./rebar -v update-deps", []), + {ok, _} = retest_sh:run("./rebar -v compile", []), + os:cmd("cp a3.rebar.config apps/a1/rebar.config"), + {ok, _} = retest_sh:run("./rebar -v update-deps", []), + {ok, _} = retest_sh:run("./rebar -v compile", []), + os:cmd("cp a4.rebar.config apps/a1/rebar.config"), + {ok, _} = retest_sh:run("./rebar -v update-deps", []), + {ok, _} = retest_sh:run("./rebar -v compile", []), + ok. + +%% +%% Generate the contents of a simple .app file +%% +app(Name, Modules, Version) -> + App = {application, Name, + [{description, atom_to_list(Name)}, + {vsn, Version}, + {modules, Modules}, + {registered, []}, + {applications, [kernel, stdlib]}]}, + io_lib:format("~p.\n", [App]). diff --git a/src/rebar_core.erl b/src/rebar_core.erl index fcfa62a..4d50f4f 100644 --- a/src/rebar_core.erl +++ b/src/rebar_core.erl @@ -163,6 +163,13 @@ skip_or_process_dir({_, ModuleSetFile}=ModuleSet, Config, CurrentCodePath, skip_or_process_dir1(AppFile, ModuleSet, Config, CurrentCodePath, Dir, Command, DirSet) -> case rebar_app_utils:is_skipped_app(Config, AppFile) of + {Config1, {true, _SkippedApp}} when Command == 'update-deps' -> + %% update-deps does its own app skipping. Unfortunately there's no + %% way to signal this to rebar_core, so we have to explicitly do it + %% here... Otherwise if you use app=, it'll skip the toplevel + %% directory and nothing will be updated. + process_dir1(Dir, Command, DirSet, Config1, + CurrentCodePath, ModuleSet); {Config1, {true, SkippedApp}} -> ?DEBUG("Skipping app: ~p~n", [SkippedApp]), Config2 = increment_operations(Config1), @@ -172,8 +179,9 @@ skip_or_process_dir1(AppFile, ModuleSet, Config, CurrentCodePath, CurrentCodePath, ModuleSet) end. -process_dir1(Dir, Command, DirSet, Config0, CurrentCodePath, +process_dir1(Dir, Command, DirSet, Config, CurrentCodePath, {DirModules, ModuleSetFile}) -> + Config0 = rebar_config:set(Config, current_command, Command), %% Get the list of modules for "any dir". This is a catch-all list %% of modules that are processed in addition to modules associated %% with this directory type. These any_dir modules are processed diff --git a/src/rebar_deps.erl b/src/rebar_deps.erl index be6283d..5e4f482 100644 --- a/src/rebar_deps.erl +++ b/src/rebar_deps.erl @@ -68,37 +68,51 @@ preprocess(Config, _) -> %% Add available deps to code path Config3 = update_deps_code_path(Config2, AvailableDeps), - %% If skip_deps=true, mark each dep dir as a skip_dir w/ the core so that - %% the current command doesn't run on the dep dir. However, pre/postprocess - %% WILL run (and we want it to) for transitivity purposes. - %% - %% Also, if skip_deps=comma,separated,app,list, then only the given - %% dependencies are skipped. - NewConfig = case rebar_config:get_global(Config3, skip_deps, false) of - "true" -> - lists:foldl( - fun(#dep{dir = Dir}, C) -> - rebar_config:set_skip_dir(C, Dir) - end, Config3, AvailableDeps); - Apps when is_list(Apps) -> - SkipApps = [list_to_atom(App) || App <- string:tokens(Apps, ",")], - lists:foldl( - fun(#dep{dir = Dir, app = App}, C) -> - case lists:member(App, SkipApps) of - true -> rebar_config:set_skip_dir(C, Dir); - false -> C - end - end, Config3, AvailableDeps); - _ -> - Config3 - end, - %% Filtering out 'raw' dependencies so that no commands other than %% deps-related can be executed on their directories. NonRawAvailableDeps = [D || D <- AvailableDeps, not D#dep.is_raw], - %% Return all the available dep directories for process - {ok, NewConfig, dep_dirs(NonRawAvailableDeps)}. + case rebar_config:get(Config, current_command, undefined) of + 'update-deps' -> + %% Skip ALL of the dep folders, we do this because we don't want + %% any other calls to preprocess() for update-deps beyond the + %% toplevel directory. They aren't actually harmful, but they slow + %% things down unnecessarily. + NewConfig = lists:foldl(fun(D, Acc) -> + rebar_config:set_skip_dir(Acc, D#dep.dir) + end, Config3, collect_deps(rebar_utils:get_cwd(), Config3)), + %% Return the empty list, as we don't want anything processed before + %% us. + {ok, NewConfig, []}; + _ -> + %% If skip_deps=true, mark each dep dir as a skip_dir w/ the core so that + %% the current command doesn't run on the dep dir. However, pre/postprocess + %% WILL run (and we want it to) for transitivity purposes. + %% + %% Also, if skip_deps=comma,separated,app,list, then only the given + %% dependencies are skipped. + NewConfig = case rebar_config:get_global(Config3, skip_deps, false) of + "true" -> + lists:foldl( + fun(#dep{dir = Dir}, C) -> + rebar_config:set_skip_dir(C, Dir) + end, Config3, AvailableDeps); + Apps when is_list(Apps) -> + SkipApps = [list_to_atom(App) || App <- string:tokens(Apps, ",")], + lists:foldl( + fun(#dep{dir = Dir, app = App}, C) -> + case lists:member(App, SkipApps) of + true -> rebar_config:set_skip_dir(C, Dir); + false -> C + end + end, Config3, AvailableDeps); + _ -> + Config3 + end, + + %% Return all the available dep directories for process + {ok, NewConfig, dep_dirs(NonRawAvailableDeps)} + end. postprocess(Config, _) -> case rebar_config:get_xconf(Config, ?MODULE, undefined) of @@ -169,17 +183,23 @@ do_check_deps(Config) -> {ok, save_dep_dirs(Config2, lists:reverse(PulledDeps))}. 'update-deps'(Config, _) -> - %% Determine what deps are required - RawDeps = rebar_config:get_local(Config, deps, []), - {Config1, Deps} = find_deps(Config, read, RawDeps), + {Config2, UpdatedDeps} = update_deps_int(rebar_config:set(Config, depowner, dict:new()), []), + DepOwners = rebar_config:get(Config2, depowner, dict:new()), - %% Update each dep - UpdatedDeps = [update_source(Config1, D) - || D <- Deps, D#dep.source =/= undefined], + %% check for conflicting deps + [?ERROR("Conflicting dependencies for ~p: ~p~n", [K, + [{"From: " ++ string:join(dict:fetch(D, + DepOwners), + ", "), + {D#dep.vsn_regex, + D#dep.source}} || D <- V]]) || + {K, V} <- dict:to_list(lists:foldl(fun(Dep, Acc) -> + dict:append(Dep#dep.app, Dep, Acc) + end, dict:new(), UpdatedDeps)), length(V) > 1], %% Add each updated dep to our list of dirs for post-processing. This yields %% the necessary transitivity of the deps - {ok, save_dep_dirs(Config1, UpdatedDeps)}. + {ok, save_dep_dirs(Config, UpdatedDeps)}. 'delete-deps'(Config, _) -> %% Delete all the available deps in our deps/ directory, if any @@ -540,7 +560,8 @@ update_source1(AppDir, {git, Url, ""}) -> update_source1(AppDir, {git, _Url, {branch, Branch}}) -> ShOpts = [{cd, AppDir}], rebar_utils:sh("git fetch origin", ShOpts), - rebar_utils:sh(?FMT("git checkout -q origin/~s", [Branch]), ShOpts); + rebar_utils:sh(?FMT("git checkout -q ~s", [Branch]), ShOpts), + rebar_utils:sh(?FMT("git pull --ff-only --no-rebase -q origin ~s", [Branch]), ShOpts); update_source1(AppDir, {git, _Url, {tag, Tag}}) -> ShOpts = [{cd, AppDir}], rebar_utils:sh("git fetch --tags origin", ShOpts), @@ -566,6 +587,81 @@ update_source1(AppDir, {fossil, _Url, Version}) -> rebar_utils:sh("fossil pull", [{cd, AppDir}]), rebar_utils:sh(?FMT("fossil update ~s", [Version]), []). +%% Recursively update deps, this is not done via rebar's usual dep traversal as +%% that is the wrong order (tips are updated before branches). Instead we do a +%% traverse the deps at each level completely before traversing *their* deps. +%% This allows updates to actually propogate down the tree, rather than fail to +%% flow up the tree, which was the previous behaviour. +update_deps_int(Config0, UDD) -> + %% Determine what deps are required + ConfDir = filename:basename(rebar_utils:get_cwd()), + RawDeps = rebar_config:get_local(Config0, deps, []), + {Config1, Deps} = find_deps(Config0, read, RawDeps), + + %% Update each dep + UpdatedDeps = [update_source(Config1, D) + || D <- Deps, D#dep.source =/= undefined, + not lists:member(D, UDD), + not should_skip_update_dep(Config1, D) + ], + + lists:foldl(fun(Dep, {Config, Updated}) -> + {true, AppDir} = get_deps_dir(Config, Dep#dep.app), + Config2 = case has_vcs_dir(element(1, Dep#dep.source), AppDir) of + false -> + %% If the dep did not exist (maybe it was added) + %% clone it. We'll traverse ITS deps below. and + %% clone them if needed. + {C1, _D1} = use_source(Config, Dep), + C1; + true -> + Config + end, + ok = file:set_cwd(AppDir), + Config3 = rebar_config:new(Config2), + %% track where a dep comes from... + Config4 = rebar_config:set(Config3, depowner, + dict:append(Dep, ConfDir, + rebar_config:get(Config3, + depowner, + dict:new()))), + + {Config5, Res} = update_deps_int(Config4, Updated), + {Config5, lists:umerge(lists:sort(Res), + lists:sort(Updated))} + end, {Config1, lists:umerge(lists:sort(UpdatedDeps), + lists:sort(UDD))}, UpdatedDeps). + +should_skip_update_dep(Config, Dep) -> + {true, AppDir} = get_deps_dir(Config, Dep#dep.app), + case rebar_app_utils:is_app_dir(AppDir) of + false -> + false; + {true, AppFile} -> + case rebar_app_utils:is_skipped_app(Config, AppFile) of + {_Config, {true, _SkippedApp}} -> + true; + _ -> + false + end + end. + +%% Recursively walk the deps and build a list of them. +collect_deps(Dir, C) -> + case file:set_cwd(Dir) of + ok -> + Config = rebar_config:new(C), + RawDeps = rebar_config:get_local(Config, deps, []), + {Config1, Deps} = find_deps(Config, read, RawDeps), + + lists:flatten(Deps ++ [begin + {true, AppDir} = get_deps_dir(Config1, Dep#dep.app), + collect_deps(AppDir, C) + end || Dep <- Deps]); + _ -> + [] + end. + %% =================================================================== %% Source helper functions