Basic implementation of port compiler is now complete

src/rebar_port_compiler.erl

@@ -33,13 +33,42 @@
 %% Public API
 %% ===================================================================
 
-%% Port driver name - determined by app name
+%% Supported configuration variables:
-%% Source files (or c_src/*.c by default)
+%%
-%% Pre-compile hook (optional)
+%% * port_sources - Erlang list of files and/or wildcard strings to be compiled
-%% Env variables
+%%
+%% * port_envs - Erlang list of key/value pairs which will control the environment when
+%%               running the compiler and linker. By default, the following variables
+%%               are defined:
+%%               CC       - C compiler
+%%               CXX      - C++ compiler
+%%               CFLAGS   - C compiler
+%%               CXXFLAGS - C++ compiler
+%%               LDFLAGS  - Link flags
+%%               DRIVER_CFLAGS  - default -I paths for erts and ei
+%%               DRIVER_LDFLAGS - default -L and -lerl_interface -lei
+%%
+%%               Note that if you wish to extend (vs. replace) these variables, you MUST
+%%               include a shell-style reference in your definition. E.g. to extend CFLAGS,
+%%               do something like:
+%%
+%%               {port_envs, [{"CFLAGS", "$CFLAGS -MyOtherOptions"}]}
+%%
+%%               It is also possible to specify platform specific options by specifying a triplet
+%%               where the first string is a regex that is checked against erlang's system architecture
+%%               string. E.g. to specify a CFLAG that only applies to x86_64 on linux do:
+%%
+%%               {port_envs, [{"x86_64.*-linux", "CFLAGS", "$CFLAGS -X86Options"}]}
+%%
+%% * port_pre_script - Tuple which specifies a pre-compilation script to run, and a filename that
+%%                     exists as a result of the script running. 
+%%
+%% * port_cleanup_script - String that specifies a script to run during cleanup. Use this to remove
+%%                         files/directories created by port_pre_script.
+%%
 
-compile(Config, _AppFile) ->
+compile(Config, AppFile) ->
-    %% Compose list of sources from config file -- default to c_src/*.c
+    %% Compose list of sources from config file -- defaults to c_src/*.c
     Sources = expand_sources(rebar_config:get_list(Config, port_sources, ["c_src/*.c"]), []),
     case Sources of
         [] ->
@@ -47,24 +76,44 @@ compile(Config, _AppFile) ->
         _ ->
             %% Extract environment values from the config (if specified) and merge with the
             %% default for this operating system. This enables max flexibility for users.
-            OperatingSystem = rebar_utils:get_os(),
+            DefaultEnvs  = filter_envs(default_env(), []),
-            DefaultEnvs  = driver_envs() ++ default_envs(OperatingSystem),
+            OverrideEnvs = filter_envs(rebar_config:get_list(Config, port_envs, []), []),
-            OverrideEnvs = rebar_config:get_list(Config, port_env, []),
             Env = merge_envs(OverrideEnvs, DefaultEnvs),
             
-            %% One or more files are available for building. Run the pre-compile hook, if necessary.
+            %% One or more files are available for building. Run the pre-compile hook, if
-%            run_precompile_hook(Config),
+            %% necessary.
+            run_precompile_hook(Config, Env),
 
             %% Compile each of the sources
-            compile_each(Sources, [], Config, Env),
+            {NewBins, ExistingBins} = compile_each(Sources, Config, Env, [], []),
-            ok
 
-            %% Finally, link everything together
+            %% Construct the driver name and make sure priv/ exists
-%            do_link(Config, AppFile, Bins)
+            SoName = so_name(AppFile),
+            ok = filelib:ensure_dir(SoName),
+
+            %% Only relink if necessary, given the SoName and list of new binaries
+            case needs_link(SoName, NewBins) of
+                true ->
+                    AllBins = string:join(NewBins ++ ExistingBins, " "),
+                    rebar_utils:sh_failfast(?FMT("$CC ~s $LDFLAGS $DRIVER_LDFLAGS -o ~s", [AllBins, SoName]), Env);
+                false ->
+                    ?INFO("Skipping relink of ~s\n", [SoName]),
+                    ok
+            end
     end.
 
-clean(Config, _AppFile) ->
+clean(Config, AppFile) ->
-    ok.
+    %% Build a list of sources so as to derive all the bins we generated
+    Sources = expand_sources(rebar_config:get_list(Config, port_sources, ["c_src/*.c"]), []),
+    rebar_file_utils:delete_each([source_to_bin(S) || S <- Sources]),
+
+    %% Delete the .so file
+    rebar_file_utils:delete_each([so_name(AppFile)]),
+
+    %% Run the cleanup script, if it exists
+    run_cleanup_hook(Config).
+
+
 
 
 %% ===================================================================
@@ -77,34 +126,50 @@ expand_sources([Spec | Rest], Acc) ->
     Acc2 = filelib:wildcard(Spec) ++ Acc,
     expand_sources(Rest, Acc2).
 
+run_precompile_hook(Config, Env) ->
+    case rebar_config:get(Config, port_pre_script, undefined) of
+        undefined ->
+            ok;
+        {Script, BypassFileName} ->
+            case filelib:is_regular(BypassFileName) of
+                false ->
+                    ?CONSOLE("Running ~s\n", [Script]),
+                    rebar_utils:sh_failfast(Script, Env);
+                true ->
+                    ?INFO("~s exists; not running ~s\n", [BypassFileName, Script])
+            end
+    end.
 
-%% CC       - C compiler
+run_cleanup_hook(Config) ->
-%% CXX      - C++ compiler
+    case rebar_config:get(Config, port_cleanup_script, undefined) of
-%% CFLAGS   - C compiler
+        undefined ->
-%% CXXFLAGS - C++ compiler
+            ok;
-%% LDFLAGS  - Link flags
+        Script ->
-
+            ?CONSOLE("Running ~s\n", [Script]),
-%% DRIVER_CFLAGS  - default -I paths for erts and ei
+            rebar_utils:sh_failfast(Script, [])
-%% DRIVER_LDFLAGS - default -L and -lerl_interface -lei 
+    end.
 
 
-compile_each([], Acc, Config, Env) ->
+compile_each([], Config, Env, NewBins, ExistingBins) ->
-    lists:reverse(Acc);
+    {lists:reverse(NewBins), lists:reverse(ExistingBins)};
-compile_each([Source | Rest], Acc, Config, Env) ->
+compile_each([Source | Rest], Config, Env, NewBins, ExistingBins) ->
     Ext = filename:extension(Source),
     Bin = filename:rootname(Source, Ext) ++ ".o",
-    ?CONSOLE("Compiling ~s\n", [Source]),
+    case needs_compile(Source, Bin) of
-    Compiler = compiler(Ext),
+        true ->
-    case compiler(Ext) of
+            ?CONSOLE("Compiling ~s\n", [Source]),
-        "$CC" ->
+            case compiler(Ext) of
-            sh(?FMT("$CC -c $CFLAGS $DRIVER_CFLAGS ~s ~s", [Source, Bin]), Env);
+                "$CC" ->
-        "$CXX" ->
+                    rebar_utils:sh_failfast(?FMT("$CC -c $CFLAGS $DRIVER_CFLAGS ~s -o ~s", [Source, Bin]), Env);
-            sh(?FMT("$CXX -c $CXXFLAGS $DRIVER_CFLAGS ~s ~s", [Source, Bin]), Env)
+                "$CXX" ->
-    end,
+                    rebar_utils:sh_failfast(?FMT("$CXX -c $CXXFLAGS $DRIVER_CFLAGS ~s -o ~s", [Source, Bin]), Env)
-    compile_each(Rest, [Bin | Acc], Config, Env).
+            end,
-
+            compile_each(Rest, Config, Env, [Bin | NewBins], ExistingBins);
-
         
+        false ->
+            ?INFO("Skipping ~s\n", [Source]),
+            compile_each(Rest, Config, Env, NewBins, [Bin | ExistingBins])
+    end.
 
           
 
@@ -112,6 +177,18 @@ needs_compile(Source, Bin) ->
     %% TODO: Generate depends using gcc -MM so we can also check for include changes
     filelib:last_modified(Bin) < filelib:last_modified(Source).
 
+needs_link(SoName, []) ->
+    filelib:last_modified(SoName) == 0;
+needs_link(SoName, NewBins) ->
+    MaxLastMod = lists:max([filelib:last_modified(B) || B <- NewBins]),
+    case filelib:last_modified(SoName) of
+        0 ->
+            true;
+        Other ->
+            ?DEBUG("Checking ~p < ~p", [MaxLastMod, Other]),
+            MaxLastMod < Other
+    end.
+    
 merge_envs(OverrideEnvs, DefaultEnvs) ->
     orddict:merge(fun(Key, Override, Default) ->
                           expand_env_variable(Override, Key, Default)
@@ -120,8 +197,9 @@ merge_envs(OverrideEnvs, DefaultEnvs) ->
                   orddict:from_list(DefaultEnvs)).
 
 
-    
+%%
-
+%% Choose a compiler variable, based on a provided extension
+%%
 compiler(".cc")  -> "$CXX";
 compiler(".cp")  -> "$CXX";
 compiler(".cxx") -> "$CXX";
@@ -131,49 +209,66 @@ compiler(".c++") -> "$CXX";
 compiler(".C")   -> "$CXX";
 compiler(_)      -> "$CC".
 
+
+%%
+%% Given env. variable FOO we want to expand all references to
+%% it in InStr. References can have two forms: $FOO and ${FOO}
+%%
 expand_env_variable(InStr, VarName, VarValue) ->
-    %% Given env. variable FOO we want to expand all references to
+
-    %% it in InStr. References can have two forms: $FOO and ${FOO}
     R1 = re:replace(InStr, "\\\$" ++ VarName, VarValue),
-    re:replace(R1, "\\\${" ++ VarName ++ "}", VarValue).
+    re:replace(R1, "\\\${" ++ VarName ++ "}", VarValue, [{return, list}]).
+
+
+%%
+%% Filter a list of env vars such that only those which match the provided
+%% architecture regex (or do not have a regex) are returned.
+%%
+filter_envs([], Acc) ->
+    lists:reverse(Acc);
+filter_envs([{ArchRegex, Key, Value} | Rest], Acc) ->
+    case rebar_utils:is_arch(ArchRegex) of
+        true ->
+            filter_envs(Rest, [{Key, Value} | Acc]);
+        false ->
+            filter_envs(Rest, Acc)
+    end;
+filter_envs([{Key, Value} | Rest], Acc) ->
+    filter_envs(Rest, [{Key, Value} | Acc]).
 
 
 erts_dir() ->
     lists:concat([code:root_dir(), "/erts-", erlang:system_info(version)]).
 
-driver_envs() ->
+default_env() ->
-    [{"DRIVER_CFLAGS", lists:concat([" -I", code:lib_dir(erl_interface, include),
+    [{"CC", "gcc"},
+     {"CXX", "g++"},
+     {"CFLAGS", "-g -Wall -fPIC"},
+     {"CXXFLAGS", "-g -Wall -fPIC"},
+     {"darwin", "LDFLAGS", "-bundle -flat_namespace -undefined suppress"},
+     {"linux", "LDFLAGS", "-shared"},
+     {"DRIVER_CFLAGS", lists:concat([" -I", code:lib_dir(erl_interface, include),
                                      " -I", filename:join(erts_dir(), include),
                                      " "])},
      {"DRIVER_LDFLAGS", lists:concat([" -L", code:lib_dir(erl_interface, lib),
                                       " -lerl_interface -lei"])}].
 
-default_envs(darwin) ->
-    [{"CC", "gcc"},
-     {"CXX", "g++"},
-     {"CFLAGS", "-g -Wall -fPIC"},
-     {"LDFLAGS", "-bundle -flat_namespace -undefined surpress"}];
-default_envs(linux) ->
-    [{"CC", "gcc"},
-     {"CXX", "g++"},
-     {"CFLAGS", "-g -Wall -fPIC"},
-     {"LDFLAGS", "-shared"}];
-default_envs(Os) ->
-    ?ERROR("Unsupported operating system ~s: can not generate default build environment.\n", [Os]),
-    ?FAIL.
 
 
-sh(Command, Env) ->
+source_to_bin(Source) ->
-    ?CONSOLE("Cmd: ~p\n~p\n", [Command, Env]),
+    Ext = filename:extension(Source),
-    Port = open_port({spawn, Command}, [{env, Env}, exit_status, {line, 16384},
+    filename:rootname(Source, Ext) ++ ".o".
-                                        use_stdio, stderr_to_stdout]),
+
-    sh_loop(Port).
+so_name(AppFile) ->
+    %% Get the app name, which we'll use to generate the linked port driver name
+    case rebar_app_utils:load_app_file(AppFile) of
+        {ok, AppName, _} ->
+            ok;
+        error ->
+            AppName = undefined,
+            ?FAIL
+    end,
+
+    %% Construct the driver name 
+    ?FMT("priv/~s_drv.so", [AppName]).
 
-sh_loop(Port) ->            
-    receive
-        {Port, {data, {_, Line}}} ->
-            ?CONSOLE("> ~s\n", [Line]),
-            sh_loop(Port);
-        {Port, Other} ->
-            ?CONSOLE(">> ~p\n", [Other])
-    end.