From 2b6fa7a25e0acdee4ce2a452152b688f2df27dea Mon Sep 17 00:00:00 2001 From: David de Boer Date: Wed, 14 Sep 2016 16:32:16 +0200 Subject: Ignore mv warnings In some cases, mv will throw a warning, while still moving the files correctly and returning a 0 return code: "mv: can't preserve ownership of ... Permission denied". --- src/rebar_file_utils.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 104c047..437780d 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -171,9 +171,14 @@ mv(Source, Dest) -> {unix, _} -> EscSource = rebar_utils:escape_chars(Source), EscDest = rebar_utils:escape_chars(Dest), - {ok, []} = rebar_utils:sh(?FMT("mv ~s ~s", [EscSource, EscDest]), - [{use_stdout, false}, abort_on_error]), - ok; + case rebar_utils:sh(?FMT("mv ~s ~s", [EscSource, EscDest]), + [{use_stdout, false}, abort_on_error]) of + {ok, []} -> + ok; + {ok, Warning} -> + ?WARN("mv: ~p", [Warning]), + ok + end; {win32, _} -> Cmd = case filelib:is_dir(Source) of true -> -- cgit v1.1 From 0afb4c4d477b1c1fba54ad3ef086d0697a5faeaa Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Tue, 11 Oct 2016 10:36:12 +0300 Subject: Made reading sys.configs consistent with OTP specification. --- src/rebar_file_utils.erl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 437780d..28cc516 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -72,11 +72,14 @@ consult_config(State, Filename) -> [T] -> T; [] -> [] end, - SubConfigs = [consult_config(State, Entry ++ ".config") || - Entry <- Config, is_list(Entry) - ], - - [Config | lists:merge(SubConfigs)]. + lists:flatmap( + fun (SubConfig) when is_list(SubConfig) -> + case consult_config(State, SubConfig) of + [] -> consult_config(State, SubConfig ++ ".config"); + X -> X + end; + (Entry) -> [Entry] + end, Config). format_error({bad_term_file, AppFile, Reason}) -> io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). -- cgit v1.1 From 4ce21e4c800108030d5c646b226032f460e68065 Mon Sep 17 00:00:00 2001 From: Alexander Sedov Date: Tue, 11 Oct 2016 16:30:27 +0300 Subject: Avoid backward-compatibility-breaking changes. --- src/rebar_file_utils.erl | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 28cc516..6721b5a 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -72,14 +72,16 @@ consult_config(State, Filename) -> [T] -> T; [] -> [] end, - lists:flatmap( - fun (SubConfig) when is_list(SubConfig) -> - case consult_config(State, SubConfig) of - [] -> consult_config(State, SubConfig ++ ".config"); - X -> X - end; - (Entry) -> [Entry] - end, Config). + JoinedConfig = lists:flatmap( + fun (SubConfig) when is_list(SubConfig) -> + case lists:suffix(".config", SubConfig) of + false -> consult_config(State, SubConfig ++ ".config"); + true -> consult_config(State, SubConfig) + end; + (Entry) -> [Entry] + end, Config), + %% Backwards compatibility + [JoinedConfig]. format_error({bad_term_file, AppFile, Reason}) -> io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). -- cgit v1.1 From e08145b498a59c1c0c100323646471ea1aa37c41 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Thu, 17 Nov 2016 08:57:16 -0500 Subject: Allow rebar3 to edoc itself --- src/rebar_file_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 6721b5a..f2467c5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -237,10 +237,10 @@ write_file_if_contents_differ(Filename, Bytes) -> %% returns an os appropriate tmpdir given a path -spec system_tmpdir() -> file:filename(). +system_tmpdir() -> system_tmpdir([]). + -spec system_tmpdir(PathComponents) -> file:filename() when PathComponents :: [file:name()]. - -system_tmpdir() -> system_tmpdir([]). system_tmpdir(PathComponents) -> Tmp = case erlang:system_info(system_architecture) of "win32" -> -- cgit v1.1 From 3da4cc222197e01886d3baaeca7a380e02ff3125 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 25 Nov 2016 21:32:43 -0500 Subject: Type specifications and edocs improvements Includes improvments and function documentation for all modules (in alphabetical order) up to rebar_core, and may have included more in other modules as I saw fit to dig and understand more of the internals. --- src/rebar_file_utils.erl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index f2467c5..8645641 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -300,9 +300,8 @@ canonical_path([_|Acc], [".."|Rest]) -> canonical_path(Acc, Rest); canonical_path([], [".."|Rest]) -> canonical_path([], Rest); canonical_path(Acc, [Component|Rest]) -> canonical_path([Component|Acc], Rest). -%% returns canonical target of path if path is a link, otherwise returns path +%% @doc returns canonical target of path if path is a link, otherwise returns path -spec resolve_link(string()) -> string(). - resolve_link(Path) -> case file:read_link(Path) of {ok, Target} -> @@ -310,9 +309,8 @@ resolve_link(Path) -> {error, _} -> Path end. -%% splits a path into dirname and basename +%% @doc splits a path into dirname and basename -spec split_dirname(string()) -> {string(), string()}. - split_dirname(Path) -> {filename:dirname(Path), filename:basename(Path)}. -- cgit v1.1 From 92e6997cf128382f7596fcc546caceee9d10a428 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 27 Jan 2017 10:56:22 -0500 Subject: Fix mv command on windows Requires changing a bunch of arguments for xerocopy since it does not allow to rename while copying. Lots of tests added --- src/rebar_file_utils.erl | 105 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 18 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 8645641..436f2d5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -185,32 +185,101 @@ mv(Source, Dest) -> ok end; {win32, _} -> - Cmd = case filelib:is_dir(Source) of - true -> - ?FMT("robocopy /move /e \"~s\" \"~s\" 1> nul", - [rebar_utils:escape_double_quotes(filename:nativename(Source)), - rebar_utils:escape_double_quotes(filename:nativename(Dest))]); - false -> - ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\" 1> nul", - [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), - rebar_utils:escape_double_quotes(filename:nativename(Dest)), - rebar_utils:escape_double_quotes(filename:basename(Source))]) - end, - Res = rebar_utils:sh(Cmd, - [{use_stdout, false}, return_on_error]), - case win32_ok(Res) of - true -> ok; + case filelib:is_dir(Source) of + true -> + SrcDir = filename:nativename(Source), + DestDir = filename:nativename(Dest), + robocopy_dir(SrcDir, DestDir); false -> - {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s~n", - [Source, Dest]))} + SrcDir = filename:nativename(filename:dirname(Source)), + SrcName = filename:basename(Source), + DestDir = filename:nativename(filename:dirname(Dest)), + DestName = filename:basename(Dest), + IsDestDir = filelib:is_dir(Dest), + if IsDestDir -> + %% if basename and target name are different because + %% we move to a directory, then just move there. + %% Similarly, if they are the same but we're going to + %% a directory, let's just do that directly. + FullDestDir = filename:nativename(Dest), + robocopy_file(SrcDir, FullDestDir, SrcName) + ; SrcName =:= DestName -> + %% if basename and target name are the same and both are files, + %% we do a regular move with robocopy without rename. + robocopy_file(SrcDir, DestDir, DestName) + ; SrcName =/= DestName-> + %% If we're moving a file and the origin and + %% destination names are different: + %% - mktmp + %% - robocopy source_dir tmp_dir srcname + %% - rename srcname destname (to avoid clobbering) + %% - robocopy tmp_dir dest_dir destname + %% - remove tmp_dir + case ec_file:insecure_mkdtemp() of + {error, _Reason} -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s (tmpdir failed)~n", + [Source, Dest]))}; + TmpPath -> + case robocopy_file(SrcDir, TmpPath, SrcName) of + {error, Reason} -> + {error, Reason}; + ok -> + TmpSrc = filename:join(TmpPath, SrcName), + TmpDst = filename:join(TmpPath, DestName), + case file:rename(TmpSrc, TmpDst) of + {error, _} -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s (via rename)~n", + [Source, Dest]))}; + ok -> + case robocopy_file(TmpPath, DestDir, DestName) of + Err = {error, _} -> Err; + OK -> rm_rf(TmpPath), OK + end + end + end + end + end + end end. +robocopy_file(SrcPath, DestPath, FileName) -> + Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\"", + [rebar_utils:escape_double_quotes(SrcPath), + rebar_utils:escape_double_quotes(DestPath), + rebar_utils:escape_double_quotes(FileName)]), + Res = rebar_utils:sh(Cmd, [{use_stdout, false}, return_on_error]), + case win32_ok(Res) of + false -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s~n", + [filename:join(SrcPath, FileName), + filename:join(DestPath, FileName)]))}; + true -> + ok + end. + +robocopy_dir(Source, Dest) -> + Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\"", + [rebar_utils:escape_double_quotes(Source), + rebar_utils:escape_double_quotes(Dest)]), + Res = rebar_utils:sh(Cmd, + [{use_stdout, false}, return_on_error]), + case win32_ok(Res) of + true -> ok; + false -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s~n", + [Source, Dest]))} + end. + win32_ok({ok, _}) -> true; win32_ok({error, {Rc, _}}) when Rc<9; Rc=:=16 -> true; win32_ok(_) -> false. + delete_each([]) -> ok; delete_each([File | Rest]) -> -- cgit v1.1 From 3f27d877e3ca1c8624bd38de6b2c4d8cf16ef65e Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 27 Jan 2017 14:46:42 -0500 Subject: Re-fix the windows file movement and clean up a bit --- src/rebar_file_utils.erl | 78 ++++++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 33 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 436f2d5..8158312 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -188,7 +188,16 @@ mv(Source, Dest) -> case filelib:is_dir(Source) of true -> SrcDir = filename:nativename(Source), - DestDir = filename:nativename(Dest), + DestDir = case filelib:is_dir(Dest) of + true -> + %% to simulate unix/posix mv, we have to replicate + %% the same directory movement by moving the whole + %% top-level directory, not just the insides + SrcName = filename:basename(Source), + filename:nativename(filename:join(Dest, SrcName)); + false -> + filename:nativename(Dest) + end, robocopy_dir(SrcDir, DestDir); false -> SrcDir = filename:nativename(filename:dirname(Source)), @@ -208,43 +217,46 @@ mv(Source, Dest) -> %% we do a regular move with robocopy without rename. robocopy_file(SrcDir, DestDir, DestName) ; SrcName =/= DestName-> - %% If we're moving a file and the origin and - %% destination names are different: - %% - mktmp - %% - robocopy source_dir tmp_dir srcname - %% - rename srcname destname (to avoid clobbering) - %% - robocopy tmp_dir dest_dir destname - %% - remove tmp_dir - case ec_file:insecure_mkdtemp() of - {error, _Reason} -> - {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s (tmpdir failed)~n", - [Source, Dest]))}; - TmpPath -> - case robocopy_file(SrcDir, TmpPath, SrcName) of - {error, Reason} -> - {error, Reason}; - ok -> - TmpSrc = filename:join(TmpPath, SrcName), - TmpDst = filename:join(TmpPath, DestName), - case file:rename(TmpSrc, TmpDst) of - {error, _} -> - {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s (via rename)~n", - [Source, Dest]))}; - ok -> - case robocopy_file(TmpPath, DestDir, DestName) of - Err = {error, _} -> Err; - OK -> rm_rf(TmpPath), OK - end - end - end - end + robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) end end end. +robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> + %% If we're moving a file and the origin and + %% destination names are different: + %% - mktmp + %% - robocopy source_dir tmp_dir srcname + %% - rename srcname destname (to avoid clobbering) + %% - robocopy tmp_dir dest_dir destname + %% - remove tmp_dir + case ec_file:insecure_mkdtemp() of + {error, _Reason} -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s (tmpdir failed)~n", + [Source, Dest]))}; + TmpPath -> + case robocopy_file(SrcDir, TmpPath, SrcName) of + {error, Reason} -> + {error, Reason}; + ok -> + TmpSrc = filename:join(TmpPath, SrcName), + TmpDst = filename:join(TmpPath, DestName), + case file:rename(TmpSrc, TmpDst) of + {error, _} -> + {error, lists:flatten( + io_lib:format("Failed to move ~s to ~s (via rename)~n", + [Source, Dest]))}; + ok -> + case robocopy_file(TmpPath, DestDir, DestName) of + Err = {error, _} -> Err; + OK -> rm_rf(TmpPath), OK + end + end + end + end. + robocopy_file(SrcPath, DestPath, FileName) -> Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\"", [rebar_utils:escape_double_quotes(SrcPath), -- cgit v1.1 From 963c49f5eb9ab5b34e1843fb43305743720917ac Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Sun, 6 Aug 2017 07:26:21 -0400 Subject: Unicode support in all the places This is done through 3 main change groups: - replacing `~s` by `~ts` in format strings, so that strings that contain unicode are properly printed rather than crashing - adding the `unicode` argument to all function of the `re` module to ensure transformations on strings containing unicode data are valid instead of crashing (see issue #1302) - replacing `ec_cnv:to_binary/1` and `ec_cnv:to_list/1` with matching functions in `rebar_utils`. The last point has been done, rather than modifying and updating erlware commons, because binary and list conversions can be a contentious subject. For example, if what is being handled is actually bytes from a given binary stream, then forcing a byte-oriented interpretation of the data can corrupt it. As such, it does not appear safe to modify erlware commons' conversion functions since it may not be safe for all its users. Instead, rebar3 reimplements a subset of them (only converting atoms and chardata, ignoring numbers) with the explicit purpose of handling unicode string data. Tests were left as unchanged as possible. This may impact the ability to run rebar3's own suites in a unicode path, but respects a principle of least change for such a large patch. --- src/rebar_file_utils.erl | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 8158312..d7716e5 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -84,7 +84,7 @@ consult_config(State, Filename) -> [JoinedConfig]. format_error({bad_term_file, AppFile, Reason}) -> - io_lib:format("Error reading file ~s: ~s", [AppFile, file:format_error(Reason)]). + io_lib:format("Error reading file ~ts: ~ts", [AppFile, file:format_error(Reason)]). symlink_or_copy(Source, Target) -> Link = case os:type() of @@ -121,7 +121,7 @@ symlink_or_copy(Source, Target) -> win32_symlink(Source, Target) -> Res = rebar_utils:sh( - ?FMT("cmd /c mklink /j \"~s\" \"~s\"", + ?FMT("cmd /c mklink /j \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(filename:nativename(Target)), rebar_utils:escape_double_quotes(filename:nativename(Source))]), [{use_stdout, false}, return_on_error]), @@ -129,7 +129,7 @@ win32_symlink(Source, Target) -> true -> ok; false -> {error, lists:flatten( - io_lib:format("Failed to symlink ~s to ~s~n", + io_lib:format("Failed to symlink ~ts to ~ts~n", [Source, Target]))} end. @@ -141,7 +141,7 @@ rm_rf(Target) -> case os:type() of {unix, _} -> EscTarget = rebar_utils:escape_chars(Target), - {ok, []} = rebar_utils:sh(?FMT("rm -rf ~s", [EscTarget]), + {ok, []} = rebar_utils:sh(?FMT("rm -rf ~ts", [EscTarget]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> @@ -161,7 +161,7 @@ cp_r(Sources, Dest) -> {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], SourceStr = string:join(EscSources, " "), - {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~s \"~s\"", + {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", [SourceStr, rebar_utils:escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), ok; @@ -176,7 +176,7 @@ mv(Source, Dest) -> {unix, _} -> EscSource = rebar_utils:escape_chars(Source), EscDest = rebar_utils:escape_chars(Dest), - case rebar_utils:sh(?FMT("mv ~s ~s", [EscSource, EscDest]), + case rebar_utils:sh(?FMT("mv ~ts ~ts", [EscSource, EscDest]), [{use_stdout, false}, abort_on_error]) of {ok, []} -> ok; @@ -234,7 +234,7 @@ robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> case ec_file:insecure_mkdtemp() of {error, _Reason} -> {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s (tmpdir failed)~n", + io_lib:format("Failed to move ~ts to ~ts (tmpdir failed)~n", [Source, Dest]))}; TmpPath -> case robocopy_file(SrcDir, TmpPath, SrcName) of @@ -246,7 +246,7 @@ robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> case file:rename(TmpSrc, TmpDst) of {error, _} -> {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s (via rename)~n", + io_lib:format("Failed to move ~ts to ~ts (via rename)~n", [Source, Dest]))}; ok -> case robocopy_file(TmpPath, DestDir, DestName) of @@ -258,7 +258,7 @@ robocopy_mv_and_rename(Source, Dest, SrcDir, SrcName, DestDir, DestName) -> end. robocopy_file(SrcPath, DestPath, FileName) -> - Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\" \"~s\"", + Cmd = ?FMT("robocopy /move /e \"~ts\" \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(SrcPath), rebar_utils:escape_double_quotes(DestPath), rebar_utils:escape_double_quotes(FileName)]), @@ -266,7 +266,7 @@ robocopy_file(SrcPath, DestPath, FileName) -> case win32_ok(Res) of false -> {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s~n", + io_lib:format("Failed to move ~ts to ~ts~n", [filename:join(SrcPath, FileName), filename:join(DestPath, FileName)]))}; true -> @@ -274,7 +274,7 @@ robocopy_file(SrcPath, DestPath, FileName) -> end. robocopy_dir(Source, Dest) -> - Cmd = ?FMT("robocopy /move /e \"~s\" \"~s\"", + Cmd = ?FMT("robocopy /move /e \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(Source), rebar_utils:escape_double_quotes(Dest)]), Res = rebar_utils:sh(Cmd, @@ -283,7 +283,7 @@ robocopy_dir(Source, Dest) -> true -> ok; false -> {error, lists:flatten( - io_lib:format("Failed to move ~s to ~s~n", + io_lib:format("Failed to move ~ts to ~ts~n", [Source, Dest]))} end. @@ -301,12 +301,19 @@ delete_each([File | Rest]) -> {error, enoent} -> delete_each(Rest); {error, Reason} -> - ?ERROR("Failed to delete file ~s: ~p\n", [File, Reason]), + ?ERROR("Failed to delete file ~ts: ~p\n", [File, Reason]), ?FAIL end. write_file_if_contents_differ(Filename, Bytes) -> - ToWrite = iolist_to_binary(Bytes), + %% first try to convert directly to binaries, + %% but if it fails, we likely contain unicode and + %% need special treatment + ToWrite = try + iolist_to_binary(Bytes) + catch + error:badarg -> unicode:characters_to_binary(Bytes) + end, case file:read_file(Filename) of {ok, ToWrite} -> ok; @@ -401,13 +408,13 @@ split_dirname(Path) -> delete_each_dir_win32([]) -> ok; delete_each_dir_win32([Dir | Rest]) -> - {ok, []} = rebar_utils:sh(?FMT("rd /q /s \"~s\"", + {ok, []} = rebar_utils:sh(?FMT("rd /q /s \"~ts\"", [rebar_utils:escape_double_quotes(filename:nativename(Dir))]), [{use_stdout, false}, return_on_error]), delete_each_dir_win32(Rest). xcopy_win32(Source,Dest)-> - %% "xcopy \"~s\" \"~s\" /q /y /e 2> nul", Changed to robocopy to + %% "xcopy \"~ts\" \"~ts\" /q /y /e 2> nul", Changed to robocopy to %% handle long names. May have issues with older windows. Cmd = case filelib:is_dir(Source) of true -> @@ -417,11 +424,11 @@ xcopy_win32(Source,Dest)-> %% must manually add the last fragment of a directory to the `Dest` %% in order to properly replicate POSIX platforms NewDest = filename:join([Dest, filename:basename(Source)]), - ?FMT("robocopy \"~s\" \"~s\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" /e /is 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(Source)), rebar_utils:escape_double_quotes(filename:nativename(NewDest))]); false -> - ?FMT("robocopy \"~s\" \"~s\" \"~s\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e /is 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), rebar_utils:escape_double_quotes(filename:nativename(Dest)), rebar_utils:escape_double_quotes(filename:basename(Source))]) @@ -432,7 +439,7 @@ xcopy_win32(Source,Dest)-> true -> ok; false -> {error, lists:flatten( - io_lib:format("Failed to copy ~s to ~s~n", + io_lib:format("Failed to copy ~ts to ~ts~n", [Source, Dest]))} end. -- cgit v1.1 From 0588936de12fe5b1607c6a23e300904e8f0fc6c0 Mon Sep 17 00:00:00 2001 From: William H Date: Thu, 10 Aug 2017 10:05:01 -0400 Subject: Support Windows with non-NTFS filesystems The code for symlink handling failed whenever a win32 platform with no symlink capability would be detected. This patch is provided by William H from the support ticket at http://www.rebar3.org/v3/discuss/598225c90365bb00144bc07f, which adds detection of failures in non-NTFS scenarios on Win32, and then copies files instead of bailing out. The end result should be appropriate support for such a platform. --- src/rebar_file_utils.erl | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index d7716e5..c860513 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -105,7 +105,7 @@ symlink_or_copy(Source, Target) -> T = unicode:characters_to_list(Target), case filelib:is_dir(S) of true -> - win32_symlink(S, T); + win32_symlink_or_copy(S, T); false -> cp_r([S], T) end; @@ -119,20 +119,48 @@ symlink_or_copy(Source, Target) -> end end. -win32_symlink(Source, Target) -> +%% @private Compatibility function for windows +win32_symlink_or_copy(Source, Target) -> Res = rebar_utils:sh( ?FMT("cmd /c mklink /j \"~ts\" \"~ts\"", [rebar_utils:escape_double_quotes(filename:nativename(Target)), rebar_utils:escape_double_quotes(filename:nativename(Source))]), [{use_stdout, false}, return_on_error]), - case win32_ok(Res) of + case win32_mklink_ok(Res, Target) of true -> ok; - false -> - {error, lists:flatten( - io_lib:format("Failed to symlink ~ts to ~ts~n", - [Source, Target]))} + false -> cp_r_win32(Source, drop_last_dir_from_path(Target)) end. +%% @private specifically pattern match against the output +%% of the windows 'mklink' shell call; different values from +%% what win32_ok/1 handles +win32_mklink_ok({ok, _}, _) -> + true; +win32_mklink_ok({error,{1,"Local NTFS volumes are required to complete the operation.\n"}}, _) -> + false; +win32_mklink_ok({error,{1,"Cannot create a file when that file already exists.\n"}}, Target) -> + % File or dir is already in place; find if it is already a symlink (true) or + % if it is a directory (copy-required; false) + is_symlink(Target); +win32_mklink_ok(_, _) -> + false. + +%% @private +is_symlink(Filename) -> + {ok, Info} = file:read_link_info(Filename), + Info#file_info.type == symlink. + +%% @private +%% drops the last 'node' of the filename, presumably the last dir such as 'src' +%% this is because cp_r_win32/2 automatically adds the dir name, to appease +%% robocopy and be more uniform with POSIX +drop_last_dir_from_path([]) -> + []; +drop_last_dir_from_path(Path) -> + case lists:droplast(filename:split(Path)) of + [] -> []; + Dirs -> filename:join(Dirs) + end. %% @doc Remove files and directories. %% Target is a single filename, directoryname or wildcard expression. -- cgit v1.1 From 8e4f90fa08bb44e041b4225c49d5a75124adcb3c Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 1 Sep 2017 12:21:25 -0700 Subject: fix sys config merging --- src/rebar_file_utils.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index c860513..4a783f2 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -75,8 +75,9 @@ consult_config(State, Filename) -> JoinedConfig = lists:flatmap( fun (SubConfig) when is_list(SubConfig) -> case lists:suffix(".config", SubConfig) of - false -> consult_config(State, SubConfig ++ ".config"); - true -> consult_config(State, SubConfig) + %% since consult_config returns a list in a list we take the head here + false -> hd(consult_config(State, SubConfig ++ ".config")); + true -> hd(consult_config(State, SubConfig)) end; (Entry) -> [Entry] end, Config), -- cgit v1.1 From 09284a8af93e742798ca61d64fb8699e18e53207 Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Fri, 13 Oct 2017 10:09:46 -0400 Subject: Avoid guessing on utf8 decoding of app files Rather than trying one method and then the other, allow the caller to specify the encoding of the expected file. All other schemes are risky and won't work well. Rollback the function's default interface to the binary format in case any plugin used it for non-unicode content, preserving backwards compat. --- src/rebar_file_utils.erl | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 4a783f2..b0755ed 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -35,6 +35,7 @@ mv/2, delete_each/1, write_file_if_contents_differ/2, + write_file_if_contents_differ/3, system_tmpdir/0, system_tmpdir/1, reset_dir/1, @@ -334,15 +335,19 @@ delete_each([File | Rest]) -> ?FAIL end. +%% @doc backwards compat layer to pre-utf8 support write_file_if_contents_differ(Filename, Bytes) -> - %% first try to convert directly to binaries, - %% but if it fails, we likely contain unicode and - %% need special treatment - ToWrite = try - iolist_to_binary(Bytes) - catch - error:badarg -> unicode:characters_to_binary(Bytes) - end, + write_file_if_contents_differ(Filename, Bytes, raw). + +%% @doc let the user pick the encoding required; there are no good +%% heuristics for data encoding +write_file_if_contents_differ(Filename, Bytes, raw) -> + write_file_if_contents_differ_(Filename, iolist_to_binary(Bytes)); +write_file_if_contents_differ(Filename, Bytes, utf8) -> + write_file_if_contents_differ_(Filename, unicode:characters_to_binary(Bytes, utf8)). + +%% @private compare raw strings and check contents +write_file_if_contents_differ_(Filename, ToWrite) -> case file:read_file(Filename) of {ok, ToWrite} -> ok; -- cgit v1.1 From 2d5cd9c00cfa4e58066b48beee4057fdd52cc7be Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Wed, 1 Nov 2017 19:38:03 -0400 Subject: OTP-21 readiness, Full Unicode support This replaces all deprecated function usage by alternative ones based on a version switch enacted at compile time, preventing all warnings. This will likely introduce some possible runtime errors in using a Rebar3 compiled on OTP-20 or OTP-21 back in versions 19 and earlier, but we can't really work around that. A bunch of dependencies have been updated to support OTP-21 without warnings as well. --- src/rebar_file_utils.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index b0755ed..7a48a6a 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -190,7 +190,7 @@ cp_r(Sources, Dest) -> case os:type() of {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], - SourceStr = string:join(EscSources, " "), + SourceStr = rebar_string:join(EscSources, " "), {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", [SourceStr, rebar_utils:escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), -- cgit v1.1 From d477715b1a869bdb112895ad28c31e5c94943f2c Mon Sep 17 00:00:00 2001 From: Fred Hebert Date: Mon, 4 Dec 2017 11:01:47 -0500 Subject: Drop the /is switch to robocopy for win7 Has no ill effect as tested on Win10 --- src/rebar_file_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 7a48a6a..bb3ca71 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -458,11 +458,11 @@ xcopy_win32(Source,Dest)-> %% must manually add the last fragment of a directory to the `Dest` %% in order to properly replicate POSIX platforms NewDest = filename:join([Dest, filename:basename(Source)]), - ?FMT("robocopy \"~ts\" \"~ts\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" /e 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(Source)), rebar_utils:escape_double_quotes(filename:nativename(NewDest))]); false -> - ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e /is 1> nul", + ?FMT("robocopy \"~ts\" \"~ts\" \"~ts\" /e 1> nul", [rebar_utils:escape_double_quotes(filename:nativename(filename:dirname(Source))), rebar_utils:escape_double_quotes(filename:nativename(Dest)), rebar_utils:escape_double_quotes(filename:basename(Source))]) -- cgit v1.1 From b3f99aa42fcbeffa8751df308cb194a470cd0e80 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 22 Apr 2018 09:09:01 +0300 Subject: ensure Dest exists before copying to it --- src/rebar_file_utils.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index bb3ca71..0dbc40a 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -191,8 +191,8 @@ cp_r(Sources, Dest) -> {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], SourceStr = rebar_string:join(EscSources, " "), - {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", - [SourceStr, rebar_utils:escape_double_quotes(Dest)]), + {ok, []} = rebar_utils:sh(?FMT("mkdir -p \"~ts\" && cp -Rp ~ts $_", + [rebar_utils:escape_double_quotes(Dest), SourceStr]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> -- cgit v1.1 From c22fde17a7a30cee5e3e5e04fab187eed2ecfe42 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Mon, 23 Apr 2018 22:34:40 +0300 Subject: fix & test --- src/rebar_file_utils.erl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 0dbc40a..492d690 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -191,8 +191,12 @@ cp_r(Sources, Dest) -> {unix, _} -> EscSources = [rebar_utils:escape_chars(Src) || Src <- Sources], SourceStr = rebar_string:join(EscSources, " "), - {ok, []} = rebar_utils:sh(?FMT("mkdir -p \"~ts\" && cp -Rp ~ts $_", - [rebar_utils:escape_double_quotes(Dest), SourceStr]), + % ensure destination exists before copying files into it + {ok, []} = rebar_utils:sh(?FMT("mkdir -p ~ts", + [rebar_utils:escape_chars(Dest)]), + [{use_stdout, false}, abort_on_error]), + {ok, []} = rebar_utils:sh(?FMT("cp -Rp ~ts \"~ts\"", + [SourceStr, rebar_utils:escape_double_quotes(Dest)]), [{use_stdout, false}, abort_on_error]), ok; {win32, _} -> -- cgit v1.1 From dec484643c233fda9c17d38c1854ba7f3f37547b Mon Sep 17 00:00:00 2001 From: Tristan Sloughter Date: Fri, 5 Oct 2018 08:29:07 -0600 Subject: compiler behaviour (#1893) * add compile type for dynamic project compilation * new rebar_compiler abstraction for running multiple compilers rebar_compiler is a new behaviour that a plugin can implement to be called on any ues of the compile provider to compile source files and keep track of their dependencies. * fix check that modules in .app modules list are from src_dirs * use project_type to find module for building projects * allow plugins to add project builders and compilers --- src/rebar_file_utils.erl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'src/rebar_file_utils.erl') diff --git a/src/rebar_file_utils.erl b/src/rebar_file_utils.erl index 492d690..a51a557 100644 --- a/src/rebar_file_utils.erl +++ b/src/rebar_file_utils.erl @@ -43,7 +43,8 @@ path_from_ancestor/2, canonical_path/1, resolve_link/1, - split_dirname/1]). + split_dirname/1, + ensure_dir/1]). -include("rebar.hrl"). @@ -386,7 +387,7 @@ reset_dir(Path) -> %% delete the directory if it exists _ = ec_file:remove(Path, [recursive]), %% recreate the directory - filelib:ensure_dir(filename:join([Path, "dummy.beam"])). + ensure_dir(Path). %% Linux touch but using erlang functions to work in bot *nix os and @@ -440,6 +441,10 @@ resolve_link(Path) -> split_dirname(Path) -> {filename:dirname(Path), filename:basename(Path)}. +-spec ensure_dir(filelib:dirname_all()) -> ok | {error, file:posix()}. +ensure_dir(Path) -> + filelib:ensure_dir(filename:join(Path, "fake_file")). + %% =================================================================== %% Internal functions %% =================================================================== @@ -505,7 +510,7 @@ cp_r_win32({true, SourceDir}, {false, DestDir}) -> false -> %% Specifying a target directory that doesn't currently exist. %% So let's attempt to create this directory - case filelib:ensure_dir(filename:join(DestDir, "dummy")) of + case ensure_dir(DestDir) of ok -> ok = xcopy_win32(SourceDir, DestDir); {error, Reason} -> -- cgit v1.1