Updated patches attached.
The new output looks like:
$ src/cp --debug src/cp file.sparse
'src/cp' -> 'file.sparse'
copy offload: yes, reflink: unsupported, sparse detection: no
$ truncate -s+1M file.sparse
$ src/cp --debug file.sparse file.sparse.cp
'file.sparse' -> 'file.sparse.cp'
copy offload: yes, reflink: unsupported, sparse detection: SEEK_HOLE
$ src/cp --reflink=never --debug file.sparse file.sparse.cp
'file.sparse' -> 'file.sparse.cp'
copy offload: avoided, reflink: no, sparse detection: SEEK_HOLE
cheers,
Pádraig
From 7e0415ed7a2f9e416b070fb5630ac4b42e6cfbcf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A1draig=20Brady?= <p...@draigbrady.com>
Date: Fri, 17 Feb 2023 13:46:13 +0000
Subject: [PATCH 1/2] cp,install,mv: add --debug to explain how a file is
copied
How a file is copied is dependent on the sparseness of the file,
what file system it is on, what file system the destination is on,
the attributes of the file, and whether they're being copied or not.
Also the --reflink and --sparse options directly impact the operation.
Given it's hard to reason about the combination of all of the above,
the --debug option is useful for users to directly identify if
copy offloading, reflinking, or sparse detection are being used.
It will also be useful for tests to directly query if
these operations are supported.
The new output looks as follows:
$ src/cp --debug src/cp file.sparse
'src/cp' -> 'file.sparse'
copy offload: yes, reflink: unsupported, sparse detection: no
$ truncate -s+1M file.sparse
$ src/cp --debug file.sparse file.sparse.cp
'file.sparse' -> 'file.sparse.cp'
copy offload: yes, reflink: unsupported, sparse detection: SEEK_HOLE
$ src/cp --reflink=never --debug file.sparse file.sparse.cp
'file.sparse' -> 'file.sparse.cp'
copy offload: avoided, reflink: no, sparse detection: SEEK_HOLE
* doc/coreutils.texi (cp invocation): Describe the --debug option.
(mv invocation): Likewise.
(install invocation): Likewise.
* src/copy.h: Add a new DEBUG member to cp_options, to control
whether to output debug info or not.
* src/copy.c (copy_debug): A new global structure to
unconditionally store debug into from the last copy_reg operations.
(copy_debug_string, emit_debug): New functions to print debug info.
* src/cp.c: if ("--debug") x->debug=true;
* src/install.c: Likewise.
* src/mv.c: Likewise.
* tests/cp/debug.sh: Add a new test.
* tests/local.mk: Reference the new test.
* NEWS: Mention the new feature.
---
NEWS | 3 ++
doc/coreutils.texi | 13 +++++++
src/copy.c | 91 +++++++++++++++++++++++++++++++++++++++++++++-
src/copy.h | 3 ++
src/cp.c | 9 +++++
src/install.c | 16 +++++++-
src/mv.c | 12 +++++-
tests/cp/debug.sh | 28 ++++++++++++++
tests/local.mk | 1 +
9 files changed, 172 insertions(+), 4 deletions(-)
create mode 100755 tests/cp/debug.sh
diff --git a/NEWS b/NEWS
index a3250161f..8d94f9e3c 100644
--- a/NEWS
+++ b/NEWS
@@ -112,6 +112,9 @@ GNU coreutils NEWS -*- outline -*-
cksum now accepts the --raw option to output a raw binary checksum.
No file name or other information is output in this mode.
+ cp, mv, and install now accept the --debug option to
+ print details on how a file is being copied.
+
factor now accepts the --exponents (-h) option to print factors
in the form p^e, rather than repeating the prime p, e times.
diff --git a/doc/coreutils.texi b/doc/coreutils.texi
index eedd7a374..8870dd828 100644
--- a/doc/coreutils.texi
+++ b/doc/coreutils.texi
@@ -8922,6 +8922,15 @@ Copy symbolic links as symbolic links rather than copying the files that
they point to, and preserve hard links between source files in the copies.
Equivalent to @option{--no-dereference --preserve=links}.
+@macro optDebugCopy
+@item --debug
+@opindex --debug
+@cindex debugging, copying
+Print extra information to stdout, explaining how files are copied.
+This option implies the @option{--verbose} option.
+@end macro
+@optDebugCopy
+
@item -f
@itemx --force
@opindex -f
@@ -9959,6 +9968,8 @@ Create any missing parent directories, giving them the default
attributes. Then create each given directory, setting their owner,
group and mode as given on the command line or to the defaults.
+@optDebugCopy
+
@item -g @var{group}
@itemx --group=@var{group}
@opindex -g
@@ -10121,6 +10132,8 @@ The program accepts the following options. Also see @ref{Common options}.
@optBackup
+@optDebugCopy
+
@item -f
@itemx --force
@opindex -f
diff --git a/src/copy.c b/src/copy.c
index 3f54bd561..ca43faac2 100644
--- a/src/copy.c
+++ b/src/copy.c
@@ -140,6 +140,60 @@ static bool owner_failure_ok (struct cp_options const *x);
static char const *top_level_src_name;
static char const *top_level_dst_name;
+enum copy_debug_val
+ {
+ COPY_DEBUG_UNKNOWN,
+ COPY_DEBUG_NO,
+ COPY_DEBUG_YES,
+ COPY_DEBUG_EXTERNAL,
+ COPY_DEBUG_AVOIDED,
+ COPY_DEBUG_UNSUPPORTED,
+ };
+
+/* debug info about the last file copy. */
+static struct copy_debug
+{
+ enum copy_debug_val offload;
+ enum copy_debug_val reflink;
+ enum copy_debug_val sparse_detection;
+} copy_debug;
+
+static const char*
+copy_debug_string (enum copy_debug_val debug_val)
+{
+ switch (debug_val)
+ {
+ case COPY_DEBUG_NO: return "no";
+ case COPY_DEBUG_YES: return "yes";
+ case COPY_DEBUG_AVOIDED: return "avoided";
+ case COPY_DEBUG_UNSUPPORTED: return "unsupported";
+ default: return "unknown";
+ }
+}
+
+static const char*
+copy_debug_sparse_string (enum copy_debug_val debug_val)
+{
+ switch (debug_val)
+ {
+ case COPY_DEBUG_NO: return "no";
+ case COPY_DEBUG_YES: return "zeros";
+ case COPY_DEBUG_EXTERNAL: return "SEEK_HOLE";
+ default: return "unknown";
+ }
+}
+
+/* Print --debug output on standard output. */
+static void
+emit_debug (const struct cp_options *x)
+{
+ if (! x->hard_link && ! x->symbolic_link)
+ printf ("copy offload: %s, reflink: %s, sparse detection: %s\n",
+ copy_debug_string (copy_debug.offload),
+ copy_debug_string (copy_debug.reflink),
+ copy_debug_sparse_string (copy_debug.sparse_detection));
+}
+
#ifndef DEV_FD_MIGHT_BE_CHR
# define DEV_FD_MIGHT_BE_CHR false
#endif
@@ -256,6 +310,9 @@ sparse_copy (int src_fd, int dest_fd, char **abuf, size_t buf_size,
*last_write_made_hole = false;
*total_n_read = 0;
+ if (copy_debug.sparse_detection == COPY_DEBUG_UNKNOWN)
+ copy_debug.sparse_detection = hole_size ? COPY_DEBUG_YES : COPY_DEBUG_NO;
+
/* If not looking for holes, use copy_file_range if functional,
but don't use if reflink disallowed as that may be implicit. */
if (!hole_size && allow_reflink)
@@ -275,10 +332,13 @@ sparse_copy (int src_fd, int dest_fd, char **abuf, size_t buf_size,
input file seems empty. */
if (*total_n_read == 0)
break;
+ copy_debug.offload = COPY_DEBUG_YES;
return true;
}
if (n_copied < 0)
{
+ copy_debug.offload = COPY_DEBUG_UNSUPPORTED;
+
if (is_CLONENOTSUP (errno))
break;
@@ -304,9 +364,13 @@ sparse_copy (int src_fd, int dest_fd, char **abuf, size_t buf_size,
return false;
}
}
+ copy_debug.offload = COPY_DEBUG_YES;
max_n_read -= n_copied;
*total_n_read += n_copied;
}
+ else
+ copy_debug.offload = COPY_DEBUG_AVOIDED;
+
bool make_hole = false;
off_t psize = 0;
@@ -480,6 +544,8 @@ lseek_copy (int src_fd, int dest_fd, char **abuf, size_t buf_size,
off_t dest_pos = 0;
bool wrote_hole_at_eof = true;
+ copy_debug.sparse_detection = COPY_DEBUG_EXTERNAL;
+
while (0 <= ext_start)
{
off_t ext_end = lseek (src_fd, ext_start, SEEK_HOLE);
@@ -1128,6 +1194,9 @@ handle_clone_fail (int dst_dirfd, char const* dst_relname,
&& unlinkat (dst_dirfd, dst_relname, 0) != 0 && errno != ENOENT)
error (0, errno, _("cannot remove %s"), quoteaf (dst_name));
+ if (! transient_failure)
+ copy_debug.reflink = COPY_DEBUG_UNSUPPORTED;
+
if (reflink_mode == REFLINK_ALWAYS || transient_failure)
return false;
@@ -1169,6 +1238,10 @@ copy_reg (char const *src_name, char const *dst_name,
bool data_copy_required = x->data_copy_required;
bool preserve_xattr = USE_XATTR & x->preserve_xattr;
+ copy_debug.offload = COPY_DEBUG_UNKNOWN;
+ copy_debug.reflink = x->reflink_mode ? COPY_DEBUG_UNKNOWN : COPY_DEBUG_NO;
+ copy_debug.sparse_detection = COPY_DEBUG_UNKNOWN;
+
source_desc = open (src_name,
(O_RDONLY | O_BINARY
| (x->dereference == DEREF_NEVER ? O_NOFOLLOW : 0)));
@@ -1305,6 +1378,8 @@ copy_reg (char const *src_name, char const *dst_name,
}
if (s == 0)
{
+ copy_debug.reflink = COPY_DEBUG_YES;
+
/* Update the clone's timestamps and permissions
as needed. */
@@ -1346,6 +1421,13 @@ copy_reg (char const *src_name, char const *dst_name,
goto close_src_desc;
}
}
+ else
+ copy_debug.reflink = COPY_DEBUG_AVOIDED;
+ }
+ else if (data_copy_required && x->reflink_mode)
+ {
+ if (! CLONE_NOOWNERCOPY)
+ copy_debug.reflink = COPY_DEBUG_AVOIDED;
}
#endif
@@ -1416,7 +1498,10 @@ copy_reg (char const *src_name, char const *dst_name,
if (data_copy_required && x->reflink_mode)
{
if (clone_file (dest_desc, source_desc) == 0)
- data_copy_required = false;
+ {
+ data_copy_required = false;
+ copy_debug.reflink = COPY_DEBUG_YES;
+ }
else
{
if (! handle_clone_fail (dst_dirfd, dst_relname, src_name, dst_name,
@@ -1523,6 +1608,10 @@ copy_reg (char const *src_name, char const *dst_name,
return_val = false;
goto close_src_and_dst_desc;
}
+
+ /* Output debug info for data copying operations. */
+ if (x->debug)
+ emit_debug (x);
}
if (x->preserve_timestamps)
diff --git a/src/copy.h b/src/copy.h
index 9d3884403..b02aa2bbb 100644
--- a/src/copy.h
+++ b/src/copy.h
@@ -242,6 +242,9 @@ struct cp_options
/* If true, display the names of the files before copying them. */
bool verbose;
+ /* If true, display details of how files were copied. */
+ bool debug;
+
/* If true, stdin is a tty. */
bool stdin_tty;
diff --git a/src/cp.c b/src/cp.c
index 74366f9ee..75ae7de47 100644
--- a/src/cp.c
+++ b/src/cp.c
@@ -62,6 +62,7 @@ enum
{
ATTRIBUTES_ONLY_OPTION = CHAR_MAX + 1,
COPY_CONTENTS_OPTION,
+ DEBUG_OPTION,
NO_PRESERVE_ATTRIBUTES_OPTION,
PARENTS_OPTION,
PRESERVE_ATTRIBUTES_OPTION,
@@ -107,6 +108,7 @@ static struct option const long_opts[] =
{"attributes-only", no_argument, NULL, ATTRIBUTES_ONLY_OPTION},
{"backup", optional_argument, NULL, 'b'},
{"copy-contents", no_argument, NULL, COPY_CONTENTS_OPTION},
+ {"debug", no_argument, NULL, DEBUG_OPTION},
{"dereference", no_argument, NULL, 'L'},
{"force", no_argument, NULL, 'f'},
{"interactive", no_argument, NULL, 'i'},
@@ -162,6 +164,9 @@ Copy SOURCE to DEST, or multiple SOURCE(s) to DIRECTORY.\n\
-b like --backup but does not accept an argument\n\
--copy-contents copy contents of special files when recursive\n\
-d same as --no-dereference --preserve=links\n\
+"), stdout);
+ fputs (_("\
+ --debug explain how a file is copied. Implies -v\n\
"), stdout);
fputs (_("\
-f, --force if an existing destination file cannot be\n\
@@ -1000,6 +1005,10 @@ main (int argc, char **argv)
x.data_copy_required = false;
break;
+ case DEBUG_OPTION:
+ x.debug = x.verbose = true;
+ break;
+
case COPY_CONTENTS_OPTION:
copy_contents = true;
break;
diff --git a/src/install.c b/src/install.c
index 41c9de306..3aa6ea92b 100644
--- a/src/install.c
+++ b/src/install.c
@@ -107,7 +107,8 @@ static char const *strip_program = "strip";
non-character as a pseudo short option, starting with CHAR_MAX + 1. */
enum
{
- PRESERVE_CONTEXT_OPTION = CHAR_MAX + 1,
+ DEBUG_OPTION = CHAR_MAX + 1,
+ PRESERVE_CONTEXT_OPTION,
STRIP_PROGRAM_OPTION
};
@@ -116,6 +117,7 @@ static struct option const long_options[] =
{"backup", optional_argument, NULL, 'b'},
{"compare", no_argument, NULL, 'C'},
{GETOPT_SELINUX_CONTEXT_OPTION_DECL},
+ {"debug", no_argument, NULL, DEBUG_OPTION},
{"directory", no_argument, NULL, 'd'},
{"group", required_argument, NULL, 'g'},
{"mode", required_argument, NULL, 'm'},
@@ -603,6 +605,11 @@ In the 4th form, create all components of the given DIRECTORY(ies).\n\
-D create all leading components of DEST except the last,\n\
or all components of --target-directory,\n\
then copy SOURCE to DEST\n\
+"), stdout);
+ fputs (_("\
+ --debug explain how a file is copied. Implies -v\n\
+"), stdout);
+ fputs (_("\
-g, --group=GROUP set group ownership, instead of process' current group\n\
-m, --mode=MODE set permission mode (as in chmod), instead of rwxr-xr-x\n\
-o, --owner=OWNER set ownership (super-user only)\n\
@@ -615,7 +622,9 @@ In the 4th form, create all components of the given DIRECTORY(ies).\n\
-S, --suffix=SUFFIX override the usual backup suffix\n\
-t, --target-directory=DIRECTORY copy all SOURCE arguments into DIRECTORY\n\
-T, --no-target-directory treat DEST as a normal file\n\
- -v, --verbose print the name of each directory as it is created\n\
+"), stdout);
+ fputs (_("\
+ -v, --verbose print the name of each created file or directory\n\
"), stdout);
fputs (_("\
--preserve-context preserve SELinux security context\n\
@@ -816,6 +825,9 @@ main (int argc, char **argv)
signal (SIGCHLD, SIG_DFL);
#endif
break;
+ case DEBUG_OPTION:
+ x.debug = x.verbose = true;
+ break;
case STRIP_PROGRAM_OPTION:
strip_program = xstrdup (optarg);
strip_program_specified = true;
diff --git a/src/mv.c b/src/mv.c
index f27a07a1c..9cea8dac6 100644
--- a/src/mv.c
+++ b/src/mv.c
@@ -48,7 +48,8 @@
non-character as a pseudo short option, starting with CHAR_MAX + 1. */
enum
{
- NO_COPY_OPTION = CHAR_MAX + 1,
+ DEBUG_OPTION = CHAR_MAX + 1,
+ NO_COPY_OPTION,
STRIP_TRAILING_SLASHES_OPTION
};
@@ -56,6 +57,7 @@ static struct option const long_options[] =
{
{"backup", optional_argument, NULL, 'b'},
{"context", no_argument, NULL, 'Z'},
+ {"debug", no_argument, NULL, DEBUG_OPTION},
{"force", no_argument, NULL, 'f'},
{"interactive", no_argument, NULL, 'i'},
{"no-clobber", no_argument, NULL, 'n'},
@@ -256,6 +258,11 @@ Rename SOURCE to DEST, or move SOURCE(s) to DIRECTORY.\n\
--backup[=CONTROL] make a backup of each existing destination file\
\n\
-b like --backup but does not accept an argument\n\
+"), stdout);
+ fputs (_("\
+ --debug explain how a file is copied. Implies -v\n\
+"), stdout);
+ fputs (_("\
-f, --force do not prompt before overwriting\n\
-i, --interactive prompt before overwrite\n\
-n, --no-clobber do not overwrite an existing file\n\
@@ -333,6 +340,9 @@ main (int argc, char **argv)
case 'n':
x.interactive = I_ALWAYS_NO;
break;
+ case DEBUG_OPTION:
+ x.debug = x.verbose = true;
+ break;
case NO_COPY_OPTION:
x.no_copy = true;
break;
diff --git a/tests/cp/debug.sh b/tests/cp/debug.sh
new file mode 100755
index 000000000..b46adc637
--- /dev/null
+++ b/tests/cp/debug.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+# Ensure that cp --debug works as documented
+
+# Copyright (C) 2023 Free Software Foundation, Inc.
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+. "${srcdir=.}/tests/init.sh"; path_prepend_ ./src
+print_ver_ cp
+
+touch file || framework_failure_
+cp --debug file file.cp >cp.out || fail=1
+grep 'copy offload:.*reflink:.*sparse detection:' cp.out || fail=1
+cp --debug --attributes-only file file.cp >cp.out || fail=1
+returns_ 1 grep 'copy offload:.*reflink:.*sparse detection:' cp.out || fail=1
+
+Exit $fail
diff --git a/tests/local.mk b/tests/local.mk
index 4c40fd115..c8db95e99 100644
--- a/tests/local.mk
+++ b/tests/local.mk
@@ -484,6 +484,7 @@ all_tests = \
tests/cp/cp-i.sh \
tests/cp/cp-mv-backup.sh \
tests/cp/cp-parents.sh \
+ tests/cp/debug.sh \
tests/cp/deref-slink.sh \
tests/cp/dir-rm-dest.sh \
tests/cp/dir-slash.sh \
--
2.26.2
From 2dd85baae88ee30edbf2ee884e1f4e43c5f18e6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A1draig=20Brady?= <p...@draigbrady.com>
Date: Thu, 23 Feb 2023 20:28:51 +0000
Subject: [PATCH 2/2] tests: determine if SEEK_HOLE is enabled
Upcomming gnulib changes may disable SEEK_HOLE
even if the system supports it, so dynamically
check if we've SEEK_HOLE enabled.
* init.cfg (seek_data_capable_): SEEK_DATA may be disabled in the build
if the system support is deemed insufficient, so also use `cp --debug`
to determine if it's enabled.
* tests/cp/sparse-2.sh: Adjust to a more general diagnostic.
* tests/cp/sparse-extents-2.sh: Likewise.
* tests/cp/sparse-extents.sh: Likewise.
* tests/cp/sparse-perf.sh: Likewise.
---
init.cfg | 12 ++++++++++++
tests/cp/sparse-2.sh | 2 +-
tests/cp/sparse-extents-2.sh | 2 +-
tests/cp/sparse-extents.sh | 2 +-
tests/cp/sparse-perf.sh | 2 +-
5 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/init.cfg b/init.cfg
index 125de7f95..55aa45e5b 100644
--- a/init.cfg
+++ b/init.cfg
@@ -546,6 +546,18 @@ require_kill_group_()
# which SEEK_DATA support exists.
seek_data_capable_()
{
+ # Check that SEEK_HOLE support is enabled
+ # Note APFS was seen to not create sparse files < 16MiB
+ if ! truncate -s16M file.sparse_; then
+ warn_ "can't create a sparse file: assuming not SEEK_DATA capable"
+ return 1
+ fi
+ if ! cp --debug --reflink=never file.sparse_ file.sparse_.cp \
+ | grep SEEK_HOLE; then
+ return 1
+ fi
+
+ # Check that SEEK_HOLE is supported on the passed file
{ python3 < /dev/null && PYTHON_=python3; } ||
{ python < /dev/null && PYTHON_=python; }
diff --git a/tests/cp/sparse-2.sh b/tests/cp/sparse-2.sh
index 852d96da7..3024055e6 100755
--- a/tests/cp/sparse-2.sh
+++ b/tests/cp/sparse-2.sh
@@ -21,7 +21,7 @@ print_ver_ cp stat dd
touch sparse_chk
seek_data_capable_ sparse_chk \
- || skip_ "this file system lacks SEEK_DATA support"
+ || skip_ "insufficient SEEK_DATA support"
# Exercise the code that handles a file ending in a hole.
printf x > k || framework_failure_
diff --git a/tests/cp/sparse-extents-2.sh b/tests/cp/sparse-extents-2.sh
index 46be86166..575dbbfc6 100755
--- a/tests/cp/sparse-extents-2.sh
+++ b/tests/cp/sparse-extents-2.sh
@@ -26,7 +26,7 @@ touch sparse_chk
if seek_data_capable_ sparse_chk && ! df -t ext3 . >/dev/null; then
: # Current partition has working extents. Good!
else
- skip_ "current file system has insufficient SEEK_DATA support"
+ skip_ "insufficient SEEK_DATA support"
# It's not; we need to create one, hence we need root access.
require_root_
diff --git a/tests/cp/sparse-extents.sh b/tests/cp/sparse-extents.sh
index 410116877..5009f098c 100755
--- a/tests/cp/sparse-extents.sh
+++ b/tests/cp/sparse-extents.sh
@@ -23,7 +23,7 @@ require_sparse_support_
touch sparse_chk || framework_failure_
seek_data_capable_ sparse_chk ||
- skip_ 'this file system lacks SEEK_DATA support'
+ skip_ 'insufficient SEEK_DATA support'
fallocate --help >/dev/null || skip_ 'The fallocate utility is required'
touch falloc.test || framework_failure_
diff --git a/tests/cp/sparse-perf.sh b/tests/cp/sparse-perf.sh
index 2ce9359be..793ecee18 100755
--- a/tests/cp/sparse-perf.sh
+++ b/tests/cp/sparse-perf.sh
@@ -28,7 +28,7 @@ timeout 10 truncate -s1T f ||
# between the creation of the file and the use of SEEK_DATA,
# for it to determine it's an empty file (return ENXIO).
seek_data_capable_ f ||
- skip_ "this file system lacks appropriate SEEK_DATA support"
+ skip_ "insufficient SEEK_DATA support"
# Nothing can read that many bytes in so little time.
timeout 10 cp --reflink=never f f2 || fail=1
--
2.26.2
