Merge pull request #142 from Vagabond/adt-update-deps

Make update-deps traverse deps breadth-first, top-down

inttest/tdeps_update/a.erl

@@ -0,0 +1,3 @@
+-module({{module}}).
+
+-include_lib("b/include/b.hrl").

inttest/tdeps_update/a.rebar.config

@@ -0,0 +1 @@
+{deps, [{b, "0.2.3", {git, "../repo/b", {tag, "0.2.3"}}}]}.

inttest/tdeps_update/a2.rebar.config

@@ -0,0 +1 @@
+{deps, [{b, "0.2.4", {git, "../repo/b", {tag, "0.2.4"}}}]}.

inttest/tdeps_update/a3.rebar.config

@@ -0,0 +1 @@
+{deps, [{b, "0.2.5", {git, "../repo/b", {tag, "0.2.5"}}}]}.

inttest/tdeps_update/a4.rebar.config

@@ -0,0 +1,4 @@
+{deps, [
+        {b, "0.2.6", {git, "../repo/b", {tag, "0.2.6"}}},
+        {f, "0.1", {git, "../repo/f", {tag, "0.1"}}}
+       ]}.

inttest/tdeps_update/b.hrl

@@ -0,0 +1 @@
+-include_lib("c/include/c.hrl").

inttest/tdeps_update/b.rebar.config

@@ -0,0 +1 @@
+{deps, [{c, "1.0", {git, "../repo/c", {tag, "1.0"}}}]}.

inttest/tdeps_update/b2.rebar.config

@@ -0,0 +1 @@
+{deps, [{c, "1.1", {git, "../repo/c", {tag, "1.1"}}}]}.

inttest/tdeps_update/b3.rebar.config

@@ -0,0 +1 @@
+{deps, [{c, "1.2", {git, "../repo/c", {tag, "1.2"}}}]}.

inttest/tdeps_update/b4.rebar.config

@@ -0,0 +1 @@
+{deps, [{c, "1.3", {git, "../repo/c", {tag, "1.3"}}}]}.

inttest/tdeps_update/c.hrl

@@ -0,0 +1 @@
+-define(HELLO, hello).

inttest/tdeps_update/c.rebar.config

@@ -0,0 +1 @@
+{deps, [{d, "0.7", {git, "../repo/d", {tag, "0.7"}}}]}.

inttest/tdeps_update/c2.hrl

@@ -0,0 +1 @@
+-include_lib("d/include/d.hrl").

inttest/tdeps_update/c2.rebar.config

@@ -0,0 +1,4 @@
+{deps, [
+    {d, "0.7", {git, "../repo/d", {tag, "0.7"}}},
+    {e, "2.0", {git, "../repo/e", {tag, "2.0"}}}
+    ]}.

inttest/tdeps_update/c3.rebar.config

@@ -0,0 +1,4 @@
+{deps, [
+    {d, "0.7", {git, "../repo/d", {tag, "0.7"}}},
+    {e, "2.1", {git, "../repo/e", {tag, "2.1"}}}
+    ]}.

inttest/tdeps_update/d.hrl

@@ -0,0 +1 @@
+-define(HELLO, hello).

inttest/tdeps_update/root.rebar.config

@@ -0,0 +1 @@
+{sub_dirs, ["apps/a1"]}.

inttest/tdeps_update/tdeps_update_rt.erl

@@ -0,0 +1,147 @@
+-module(tdeps_update_rt).
+
+-compile(export_all).
+
+%% Exercises update deps, with recursive dependency updates.
+%% Initially:
+%%     A(v0.5) -> B(v0.2.3) -> C(v1.0)
+%% But after updating A to 0.6:
+%%     A(v0.6) -> B(v0.2.4) -> C(v1.1)
+%%                          -> D(v0.7)
+%% And after updating A to 0.7:
+%%     A(v0.7) -> B(v0.2.5) -> C(v1.2) -> E(v2.0)
+%%                          -> D(v0.7)
+%% And after updating A to 0.8:
+%%     A(v0.8) -> B(v0.2.6) -> C(v1.3) -> E(v2.1)
+%%                          -> D(v0.7)
+%%             -> F(v0.1)   -> E(v2.1)
+files() ->
+    [
+     %% A1 application
+     {create, "apps/a1/ebin/a1.app", app(a1, [a1], "0.5")},
+     {copy, "a.rebar.config", "apps/a1/rebar.config"},
+     {template, "a.erl", "apps/a1/src/a1.erl", dict:from_list([{module, a1}])},
+
+     {copy, "root.rebar.config", "rebar.config"},
+     {copy, "../../rebar", "rebar"},
+
+     %% B application
+     {create, "repo/b/ebin/b.app", app(b, [], "0.2.3")},
+     {create, "b2.app", app(b, [], "0.2.4")},
+     {create, "b3.app", app(b, [], "0.2.5")},
+     {create, "b4.app", app(b, [], "0.2.6")},
+     {copy, "b.rebar.config", "repo/b/rebar.config"},
+     {copy, "b.hrl", "repo/b/include/b.hrl"},
+
+     %% C application
+     {create, "repo/c/ebin/c.app", app(c, [], "1.0")},
+     {create, "c2.app", app(c, [], "1.1")},
+     {create, "c3.app", app(c, [], "1.2")},
+     {create, "c4.app", app(c, [], "1.3")},
+     {copy, "c.hrl", "repo/c/include/c.hrl"},
+
+     %% D application
+     {create, "repo/d/ebin/d.app", app(d, [], "0.7")},
+     {copy, "d.hrl", "repo/d/include/d.hrl"},
+
+     %% E application
+     {create, "repo/e/ebin/e.app", app(e, [], "2.0")},
+     {create, "e2.app", app(e, [], "2.1")},
+
+     %% F application
+     {create, "repo/f/ebin/f.app", app(f, [], "0.1")},
+
+     %% update files
+     {copy, "a2.rebar.config", "a2.rebar.config"},
+     {copy, "a3.rebar.config", "a3.rebar.config"},
+     {copy, "a4.rebar.config", "a4.rebar.config"},
+     {copy, "b2.rebar.config", "b2.rebar.config"},
+     {copy, "b3.rebar.config", "b3.rebar.config"},
+     {copy, "b4.rebar.config", "b4.rebar.config"},
+     {copy, "c2.hrl", "c2.hrl"},
+     {copy, "c.rebar.config", "c.rebar.config"},
+     {copy, "c2.rebar.config", "c2.rebar.config"},
+     {copy, "c3.rebar.config", "c3.rebar.config"}
+    ].
+
+apply_cmds([], _Params) ->
+    ok;
+apply_cmds([Cmd | Rest], Params) ->
+    io:format("Running: ~s (~p)\n", [Cmd, Params]),
+    {ok, _} = retest_sh:run(Cmd, Params),
+    apply_cmds(Rest, Params).
+
+run(_Dir) ->
+    %% Initialize the b/c/d apps as git repos so that dependencies pull
+    %% properly
+    GitCmds = ["git init",
+               "git add -A",
+               "git config user.email 'tdeps@example.com'",
+               "git config user.name 'tdeps'",
+               "git commit -a -m 'Initial Commit'"],
+    BCmds = ["git tag 0.2.3",
+             "cp ../../b2.rebar.config rebar.config",
+             "cp ../../b2.app ebin/b.app",
+             "git commit -a -m 'update to 0.2.4'",
+             "git tag 0.2.4",
+             "cp ../../b3.rebar.config rebar.config",
+             "cp ../../b3.app ebin/b.app",
+             "git commit -a -m 'update to 0.2.5'",
+             "git tag 0.2.5",
+             "cp ../../b4.rebar.config rebar.config",
+             "cp ../../b4.app ebin/b.app",
+             "git commit -a -m 'update to 0.2.6'",
+             "git tag 0.2.6"],
+             %"git checkout 0.2.3"],
+    CCmds = ["git tag 1.0",
+             "cp ../../c2.hrl include/c.hrl",
+             "cp ../../c2.app ebin/c.app",
+             "cp ../../c.rebar.config rebar.config",
+             "git add rebar.config",
+             "git commit -a -m 'update to 1.1'",
+             "git tag 1.1",
+             "cp ../../c3.app ebin/c.app",
+             "cp ../../c2.rebar.config rebar.config",
+             "git commit -a -m 'update to 1.2'",
+             "git tag 1.2",
+             "cp ../../c4.app ebin/c.app",
+             "cp ../../c3.rebar.config rebar.config",
+             "git commit -a -m 'update to 1.3'",
+             "git tag 1.3"],
+             %"git checkout 1.0"],
+    DCmds = ["git tag 0.7"],
+    ECmds = ["git tag 2.0",
+             "cp ../../e2.app ebin/e.app",
+             "git commit -a -m 'update to 2.1'",
+             "git tag 2.1"],
+    FCmds = ["git tag 0.1"],
+
+    ok = apply_cmds(GitCmds++BCmds, [{dir, "repo/b"}]),
+    ok = apply_cmds(GitCmds++CCmds, [{dir, "repo/c"}]),
+    ok = apply_cmds(GitCmds++DCmds, [{dir, "repo/d"}]),
+    ok = apply_cmds(GitCmds++ECmds, [{dir, "repo/e"}]),
+    ok = apply_cmds(GitCmds++FCmds, [{dir, "repo/f"}]),
+
+    {ok, _} = retest_sh:run("./rebar -v get-deps compile", []),
+    os:cmd("cp a2.rebar.config apps/a1/rebar.config"),
+    {ok, _} = retest_sh:run("./rebar -v update-deps", []),
+    {ok, _} = retest_sh:run("./rebar -v compile", []),
+    os:cmd("cp a3.rebar.config apps/a1/rebar.config"),
+    {ok, _} = retest_sh:run("./rebar -v update-deps", []),
+    {ok, _} = retest_sh:run("./rebar -v compile", []),
+    os:cmd("cp a4.rebar.config apps/a1/rebar.config"),
+    {ok, _} = retest_sh:run("./rebar -v update-deps", []),
+    {ok, _} = retest_sh:run("./rebar -v compile", []),
+    ok.
+
+%%
+%% Generate the contents of a simple .app file
+%%
+app(Name, Modules, Version) ->
+    App = {application, Name,
+           [{description, atom_to_list(Name)},
+            {vsn, Version},
+            {modules, Modules},
+            {registered, []},
+            {applications, [kernel, stdlib]}]},
+    io_lib:format("~p.\n", [App]).

src/rebar_core.erl

@@ -163,6 +163,13 @@ skip_or_process_dir({_, ModuleSetFile}=ModuleSet, Config, CurrentCodePath,
 skip_or_process_dir1(AppFile, ModuleSet, Config, CurrentCodePath,
                      Dir, Command, DirSet) ->
     case rebar_app_utils:is_skipped_app(Config, AppFile) of
+        {Config1, {true, _SkippedApp}} when Command == 'update-deps' ->
+            %% update-deps does its own app skipping. Unfortunately there's no
+            %% way to signal this to rebar_core, so we have to explicitly do it
+            %% here... Otherwise if you use app=, it'll skip the toplevel
+            %% directory and nothing will be updated.
+            process_dir1(Dir, Command, DirSet, Config1,
+                         CurrentCodePath, ModuleSet);
         {Config1, {true, SkippedApp}} ->
             ?DEBUG("Skipping app: ~p~n", [SkippedApp]),
             Config2 = increment_operations(Config1),
@@ -172,8 +179,9 @@ skip_or_process_dir1(AppFile, ModuleSet, Config, CurrentCodePath,
                          CurrentCodePath, ModuleSet)
     end.
 
-process_dir1(Dir, Command, DirSet, Config0, CurrentCodePath,
+process_dir1(Dir, Command, DirSet, Config, CurrentCodePath,
              {DirModules, ModuleSetFile}) ->
+    Config0 = rebar_config:set(Config, current_command, Command),
     %% Get the list of modules for "any dir". This is a catch-all list
     %% of modules that are processed in addition to modules associated
     %% with this directory type. These any_dir modules are processed

src/rebar_deps.erl

@@ -68,37 +68,51 @@ preprocess(Config, _) ->
     %% Add available deps to code path
     Config3 = update_deps_code_path(Config2, AvailableDeps),
 
-    %% If skip_deps=true, mark each dep dir as a skip_dir w/ the core so that
-    %% the current command doesn't run on the dep dir. However, pre/postprocess
-    %% WILL run (and we want it to) for transitivity purposes.
-    %%
-    %% Also, if skip_deps=comma,separated,app,list, then only the given
-    %% dependencies are skipped.
-    NewConfig = case rebar_config:get_global(Config3, skip_deps, false) of
-        "true" ->
-            lists:foldl(
-                fun(#dep{dir = Dir}, C) ->
-                        rebar_config:set_skip_dir(C, Dir)
-                end, Config3, AvailableDeps);
-        Apps when is_list(Apps) ->
-            SkipApps = [list_to_atom(App) || App <- string:tokens(Apps, ",")],
-            lists:foldl(
-                fun(#dep{dir = Dir, app = App}, C) ->
-                        case lists:member(App, SkipApps) of
-                            true -> rebar_config:set_skip_dir(C, Dir);
-                            false -> C
-                        end
-                end, Config3, AvailableDeps);
-        _ ->
-            Config3
-    end,
-
     %% Filtering out 'raw' dependencies so that no commands other than
     %% deps-related can be executed on their directories.
     NonRawAvailableDeps = [D || D <- AvailableDeps, not D#dep.is_raw],
 
-    %% Return all the available dep directories for process
+    case rebar_config:get(Config, current_command, undefined) of
-    {ok, NewConfig, dep_dirs(NonRawAvailableDeps)}.
+        'update-deps' ->
+            %% Skip ALL of the dep folders, we do this because we don't want
+            %% any other calls to preprocess() for update-deps beyond the
+            %% toplevel directory. They aren't actually harmful, but they slow
+            %% things down unnecessarily.
+            NewConfig = lists:foldl(fun(D, Acc) ->
+                                             rebar_config:set_skip_dir(Acc, D#dep.dir)
+                                     end, Config3, collect_deps(rebar_utils:get_cwd(), Config3)),
+            %% Return the empty list, as we don't want anything processed before
+            %% us.
+            {ok, NewConfig, []};
+        _ ->
+            %% If skip_deps=true, mark each dep dir as a skip_dir w/ the core so that
+            %% the current command doesn't run on the dep dir. However, pre/postprocess
+            %% WILL run (and we want it to) for transitivity purposes.
+            %%
+            %% Also, if skip_deps=comma,separated,app,list, then only the given
+            %% dependencies are skipped.
+            NewConfig = case rebar_config:get_global(Config3, skip_deps, false) of
+                "true" ->
+                    lists:foldl(
+                        fun(#dep{dir = Dir}, C) ->
+                                rebar_config:set_skip_dir(C, Dir)
+                        end, Config3, AvailableDeps);
+                Apps when is_list(Apps) ->
+                    SkipApps = [list_to_atom(App) || App <- string:tokens(Apps, ",")],
+                    lists:foldl(
+                        fun(#dep{dir = Dir, app = App}, C) ->
+                                case lists:member(App, SkipApps) of
+                                    true -> rebar_config:set_skip_dir(C, Dir);
+                                    false -> C
+                                end
+                        end, Config3, AvailableDeps);
+                _ ->
+                    Config3
+            end,
+
+            %% Return all the available dep directories for process
+            {ok, NewConfig, dep_dirs(NonRawAvailableDeps)}
+    end.
 
 postprocess(Config, _) ->
     case rebar_config:get_xconf(Config, ?MODULE, undefined) of
@@ -169,17 +183,23 @@ do_check_deps(Config) ->
     {ok, save_dep_dirs(Config2, lists:reverse(PulledDeps))}.
 
 'update-deps'(Config, _) ->
-    %% Determine what deps are required
+    {Config2, UpdatedDeps} = update_deps_int(rebar_config:set(Config, depowner, dict:new()), []),
-    RawDeps = rebar_config:get_local(Config, deps, []),
+    DepOwners = rebar_config:get(Config2, depowner, dict:new()),
-    {Config1, Deps} = find_deps(Config, read, RawDeps),
 
-    %% Update each dep
+    %% check for conflicting deps
-    UpdatedDeps = [update_source(Config1, D)
+    [?ERROR("Conflicting dependencies for ~p: ~p~n", [K,
-                   || D <- Deps, D#dep.source =/= undefined],
+                                                      [{"From: " ++ string:join(dict:fetch(D,
+                                                                               DepOwners),
+                                                                               ", "),
+                                                        {D#dep.vsn_regex,
+                                                         D#dep.source}} || D <- V]]) ||
+           {K, V} <- dict:to_list(lists:foldl(fun(Dep, Acc) ->
+                                                      dict:append(Dep#dep.app, Dep, Acc)
+                                              end, dict:new(), UpdatedDeps)), length(V) > 1],
 
     %% Add each updated dep to our list of dirs for post-processing. This yields
     %% the necessary transitivity of the deps
-    {ok, save_dep_dirs(Config1, UpdatedDeps)}.
+    {ok, save_dep_dirs(Config, UpdatedDeps)}.
 
 'delete-deps'(Config, _) ->
     %% Delete all the available deps in our deps/ directory, if any
@@ -540,7 +560,8 @@ update_source1(AppDir, {git, Url, ""}) ->
 update_source1(AppDir, {git, _Url, {branch, Branch}}) ->
     ShOpts = [{cd, AppDir}],
     rebar_utils:sh("git fetch origin", ShOpts),
-    rebar_utils:sh(?FMT("git checkout -q origin/~s", [Branch]), ShOpts);
+    rebar_utils:sh(?FMT("git checkout -q ~s", [Branch]), ShOpts),
+    rebar_utils:sh(?FMT("git pull --ff-only --no-rebase -q origin ~s", [Branch]), ShOpts);
 update_source1(AppDir, {git, _Url, {tag, Tag}}) ->
     ShOpts = [{cd, AppDir}],
     rebar_utils:sh("git fetch --tags origin", ShOpts),
@@ -566,6 +587,81 @@ update_source1(AppDir, {fossil, _Url, Version}) ->
     rebar_utils:sh("fossil pull", [{cd, AppDir}]),
     rebar_utils:sh(?FMT("fossil update ~s", [Version]), []).
 
+%% Recursively update deps, this is not done via rebar's usual dep traversal as
+%% that is the wrong order (tips are updated before branches). Instead we do a
+%% traverse the deps at each level completely before traversing *their* deps.
+%% This allows updates to actually propogate down the tree, rather than fail to
+%% flow up the tree, which was the previous behaviour.
+update_deps_int(Config0, UDD) ->
+    %% Determine what deps are required
+    ConfDir = filename:basename(rebar_utils:get_cwd()),
+    RawDeps = rebar_config:get_local(Config0, deps, []),
+    {Config1, Deps} = find_deps(Config0, read, RawDeps),
+
+    %% Update each dep
+    UpdatedDeps = [update_source(Config1, D)
+                   || D <- Deps, D#dep.source =/= undefined,
+                      not lists:member(D, UDD),
+                      not should_skip_update_dep(Config1, D)
+                  ],
+
+    lists:foldl(fun(Dep, {Config, Updated}) ->
+                        {true, AppDir} = get_deps_dir(Config, Dep#dep.app),
+                        Config2 = case has_vcs_dir(element(1, Dep#dep.source), AppDir) of
+                            false ->
+                                %% If the dep did not exist (maybe it was added)
+                                %% clone it. We'll traverse ITS deps below. and
+                                %% clone them if needed.
+                                {C1, _D1} = use_source(Config, Dep),
+                                C1;
+                            true ->
+                                Config
+                        end,
+                        ok = file:set_cwd(AppDir),
+                        Config3 = rebar_config:new(Config2),
+                        %% track where a dep comes from...
+                        Config4 = rebar_config:set(Config3, depowner,
+                                                   dict:append(Dep, ConfDir,
+                                                               rebar_config:get(Config3,
+                                                                                depowner,
+                                                                                dict:new()))),
+
+                        {Config5, Res} = update_deps_int(Config4, Updated),
+                        {Config5, lists:umerge(lists:sort(Res),
+                                               lists:sort(Updated))}
+                end, {Config1, lists:umerge(lists:sort(UpdatedDeps),
+                                            lists:sort(UDD))}, UpdatedDeps).
+
+should_skip_update_dep(Config, Dep) ->
+    {true, AppDir} = get_deps_dir(Config, Dep#dep.app),
+    case rebar_app_utils:is_app_dir(AppDir) of
+        false ->
+            false;
+        {true, AppFile} ->
+            case rebar_app_utils:is_skipped_app(Config, AppFile) of
+                {_Config, {true, _SkippedApp}} ->
+                    true;
+                _ ->
+                    false
+            end
+    end.
+
+%% Recursively walk the deps and build a list of them.
+collect_deps(Dir, C) ->
+    case file:set_cwd(Dir) of
+        ok ->
+            Config = rebar_config:new(C),
+            RawDeps = rebar_config:get_local(Config, deps, []),
+            {Config1, Deps} = find_deps(Config, read, RawDeps),
+
+            lists:flatten(Deps ++ [begin
+                                       {true, AppDir} = get_deps_dir(Config1, Dep#dep.app),
+                                       collect_deps(AppDir, C)
+                                   end || Dep <- Deps]);
+        _ ->
+            []
+    end.
+
 
 %% ===================================================================
 %% Source helper functions