Re: [gentoo-portage-dev] [PATCH] movefile: support in-kernel file copying on Linux (bug 607868)

[Michał Górny]

W dniu 01.03.2017, śro o godzinie 19∶03 -0800, użytkownik Zac Medico
napisał:
> Perform in-kernel file copying when possible, and also support
> reflinks and sparse files. If the optimized implementation
> fails at runtime, gracefully fallback to shutil.copyfile.
>
> Compile-time and run-time fallbacks are implemented, so that
> any incompatiblities will be handled gracefully. For example,
> if the code is compiled on a system that supports the
> copy_file_range syscall, but at run-time an older kernel that
> does not support this syscall is detected, it will be handled
> gracefully.
>
> X-Gentoo-Bug: 607868
> X-Gentoo-Bug-Url: https://bugs.gentoo.org/show_bug.cgi?id=607868
> ---
>  pym/portage/tests/util/file_copy/__init__.py      |   0
>  pym/portage/tests/util/file_copy/__test__.py      |   0
>  pym/portage/tests/util/file_copy/test_copyfile.py |  68 +++++++
>  pym/portage/util/file_copy/__init__.py            |  78 ++++++++
>  pym/portage/util/movefile.py                      |   4 +-
>  setup.py                                          |   9 +
>  src/portage_util_file_copy_reflink_linux.c        | 225 ++++++++++++++++++++++
>  7 files changed, 383 insertions(+), 1 deletion(-)
>  create mode 100644 pym/portage/tests/util/file_copy/__init__.py
>  create mode 100644 pym/portage/tests/util/file_copy/__test__.py
>  create mode 100644 pym/portage/tests/util/file_copy/test_copyfile.py
>  create mode 100644 pym/portage/util/file_copy/__init__.py
>  create mode 100644 src/portage_util_file_copy_reflink_linux.c
>
> diff --git a/pym/portage/tests/util/file_copy/__init__.py b/pym/portage/tests/util/file_copy/__init__.py
> new file mode 100644
> index 0000000..e69de29
> diff --git a/pym/portage/tests/util/file_copy/__test__.py b/pym/portage/tests/util/file_copy/__test__.py
> new file mode 100644
> index 0000000..e69de29
> diff --git a/pym/portage/tests/util/file_copy/test_copyfile.py b/pym/portage/tests/util/file_copy/test_copyfile.py
> new file mode 100644
> index 0000000..987a701
> --- /dev/null
> +++ b/pym/portage/tests/util/file_copy/test_copyfile.py
> @@ -0,0 +1,68 @@
> +# Copyright 2017 Gentoo Foundation
> +# Distributed under the terms of the GNU General Public License v2
> +
> +import shutil
> +import tempfile
> +
> +from portage import os
> +from portage.tests import TestCase
> +from portage.checksum import perform_md5
> +from portage.util.file_copy import copyfile
> +
> +
> +class CopyFileTestCase(TestCase):
> +
> +	def testCopyFile(self):
> +
> +		tempdir = tempfile.mkdtemp()
> +		try:
> +			src_path = os.path.join(tempdir, 'src')
> +			dest_path = os.path.join(tempdir, 'dest')
> +			content = b'foo'
> +
> +			with open(src_path, 'wb') as f:
> +				f.write(content)
> +
> +			copyfile(src_path, dest_path)
> +
> +			self.assertEqual(perform_md5(src_path), perform_md5(dest_path))
> +		finally:
> +			shutil.rmtree(tempdir)
> +
> +
> +class CopyFileSparseTestCase(TestCase):
> +
> +	def testCopyFileSparse(self):
> +
> +		# This test is expected to fail on platforms where we have
> +		# not implemented sparse copy, so set the todo flag in order
> +		# to tolerate failures.
> +		self.todo = True
> +
> +		tempdir = tempfile.mkdtemp()
> +		try:
> +			src_path = os.path.join(tempdir, 'src')
> +			dest_path = os.path.join(tempdir, 'dest')
> +			content = b'foo'
> +
> +			# Use seek to create some sparse blocks. Don't make these
> +			# files too big, in case the filesystem doesn't support
> +			# sparse files.
> +			with open(src_path, 'wb') as f:
> +				f.write(content)
> +				f.seek(2**18, 1)
> +				f.write(content)
> +				f.seek(2**19, 1)
> +				f.write(content)
> +
> +			copyfile(src_path, dest_path)
> +
> +			self.assertEqual(perform_md5(src_path), perform_md5(dest_path))
> +
> +			# If sparse blocks were preserved, then both files should
> +			# consume the same number of blocks.
> +			self.assertEqual(
> +				os.stat(src_path).st_blocks,
> +				os.stat(dest_path).st_blocks)
> +		finally:
> +			shutil.rmtree(tempdir)
> diff --git a/pym/portage/util/file_copy/__init__.py b/pym/portage/util/file_copy/__init__.py
> new file mode 100644
> index 0000000..5c7aff1
> --- /dev/null
> +++ b/pym/portage/util/file_copy/__init__.py
> @@ -0,0 +1,78 @@
> +# Copyright 2017 Gentoo Foundation
> +# Distributed under the terms of the GNU General Public License v2
> +
> +import os
> +import shutil
> +import tempfile
> +
> +try:
> +	from portage.util.file_copy.reflink_linux import file_copy as _file_copy
> +except ImportError:
> +	_file_copy = None
> +
> +
> +_copyfile = None
> +
> +
> +def _optimized_copyfile(src, dst):
> +	with open(src, 'rb', buffering=0) as src_file, \
> +		open(dst, 'wb', buffering=0) as dst_file:
> +		_file_copy(src_file.fileno(), dst_file.fileno())
> +
> +
> +def _test_optimized_copyfile():
> +	"""
> +	Test if _optimized_copyfile works. It will fail for Linux versions
> +	from 2.6.0 to 2.6.32, because sendfile does not support writing
> +	to regular files. It will also fail for Linux versions less than
> +	3.1 if portage was compiled 3.1 or later, due to missing support
> +	for lseek SEEK_DATA/SEEK_HOLE.

I don't really like the idea of relying on one-time attempt to determine
whether a complex mechanism lacking proper fallback will work. But I
guess you're only aiming at catching corner cases here.

> +	"""
> +	tempdir = tempfile.mkdtemp()
> +	try:
> +		src_path = os.path.join(tempdir, 'src')
> +		dest_path = os.path.join(tempdir, 'dest')
> +		content = b'foo'
> +
> +		with open(src_path, 'wb') as f:
> +			f.write(content)
> +
> +		try:
> +			_optimized_copyfile(src_path, dest_path)
> +		except Exception:
> +			return False
> +
> +		with open(dest_path, 'rb') as dest_file:
> +			if content != dest_file.read():
> +				return False
> +	finally:
> +		shutil.rmtree(tempdir)
> +
> +	return True
> +
> +
> +def copyfile(src, dst):
> +	"""
> +	Copy the contents (no metadata) of the file named src to a file
> +	named dst.
> +
> +	If possible, copying is done within the kernel, and uses
> +	"copy acceleration" techniques (such as reflinks). This also
> +	supports sparse files.
> +
> +	@param src: path of source file
> +	@type src: str
> +	@param dst: path of destination file
> +	@type dst: str
> +	"""
> +	global _copyfile
> +
> +	if _copyfile is None:
> +		if _file_copy is None:
> +			_copyfile = shutil.copyfile
> +		elif _test_optimized_copyfile():
> +			_copyfile = _optimized_copyfile
> +		else:
> +			_copyfile = shutil.copyfile

This logic looks a bit around. Wouldn't it be simpler if
_test_optimized_copyfile() just returned False is _file_copy is None?

> +
> +	_copyfile(src, dst)
> diff --git a/pym/portage/util/movefile.py b/pym/portage/util/movefile.py
> index 4be1c3b..88b35d3 100644
> --- a/pym/portage/util/movefile.py
> +++ b/pym/portage/util/movefile.py
> @@ -23,6 +23,8 @@ from portage.localization import _
>  from portage.process import spawn
>  from portage.util import writemsg
>  from portage.util._xattr import xattr
> +from portage.util.file_copy import copyfile
> +
>
>  def _apply_stat(src_stat, dest):
>  	_os.chown(dest, src_stat.st_uid, src_stat.st_gid)
> @@ -114,7 +116,7 @@ def movefile(src, dest, newmtime=None, sstat=None, mysettings=None,
>  		_copyfile = selinux.copyfile
>  		_rename = selinux.rename
>  	else:
> -		_copyfile = _shutil.copyfile
> +		_copyfile = copyfile
>  		_rename = _os.rename
>
>  	lchown = _unicode_func_wrapper(portage.data.lchown, encoding=encoding)
> diff --git a/setup.py b/setup.py
> index a346bd4..b624767 100755
> --- a/setup.py
> +++ b/setup.py
> @@ -23,6 +23,7 @@ import collections
>  import glob
>  import os
>  import os.path
> +import platform
>  import re
>  import subprocess
>  import sys
> @@ -54,6 +55,14 @@ x_c_helpers = {
>  	],
>  }
>
> +if platform.system() == 'Linux':
> +	x_c_helpers.update({
> +		'portage.util.file_copy.reflink_linux': [
> +			'src/portage_util_file_copy_reflink_linux.c',
> +		],
> +	})
> +
> +
>  class x_build(build):
>  	""" Build command with extra build_man call. """
>
> diff --git a/src/portage_util_file_copy_reflink_linux.c b/src/portage_util_file_copy_reflink_linux.c
> new file mode 100644
> index 0000000..7139c7a
> --- /dev/null
> +++ b/src/portage_util_file_copy_reflink_linux.c
> @@ -0,0 +1,225 @@
> +/* Copyright 2017 Gentoo Foundation
> + * Distributed under the terms of the GNU General Public License v2
> + */
> +
> +#include <Python.h>
> +#include <errno.h>
> +#include <stdlib.h>
> +#include <ctype.h>
> +#include <sys/sendfile.h>
> +#include <sys/stat.h>
> +#include <sys/syscall.h>
> +#include <sys/types.h>
> +#include <unistd.h>
> +
> +static PyObject * _reflink_linux_file_copy(PyObject *, PyObject *);
> +
> +static PyMethodDef reflink_linuxMethods[] = {
> +    {
> +            "file_copy",
> +            _reflink_linux_file_copy,
> +            METH_VARARGS,
> +            "Copy between two file descriptors, "
> +            "with reflink and sparse file support."
> +    },
> +    {NULL, NULL, 0, NULL}
> +};
> +
> +#if PY_MAJOR_VERSION >= 3
> +static struct PyModuleDef moduledef = {
> +    PyModuleDef_HEAD_INIT,
> +    "reflink_linux",                                /* m_name */
> +    "Module for reflink_linux copy operations",     /* m_doc */
> +    -1,                                             /* m_size */
> +    reflink_linuxMethods,                           /* m_methods */
> +    NULL,                                           /* m_reload */
> +    NULL,                                           /* m_traverse */
> +    NULL,                                           /* m_clear */
> +    NULL,                                           /* m_free */
> +};
> +
> +PyMODINIT_FUNC
> +PyInit_reflink_linux(void)
> +{
> +    PyObject *m;
> +    m = PyModule_Create(&moduledef);
> +    return m;
> +}
> +#else
> +PyMODINIT_FUNC
> +initreflink_linux(void)
> +{
> +    Py_InitModule("reflink_linux", reflink_linuxMethods);
> +}
> +#endif
> +
> +
> +static ssize_t
> +cfr_wrapper(int fd_out, int fd_in, loff_t *off_out, size_t len)
> +{
> +#ifdef __NR_copy_file_range
> +    return syscall(__NR_copy_file_range, fd_in, NULL, fd_out,
> +                   off_out, len, 0);
> +#else
> +    errno = ENOSYS;
> +    return -1;
> +#endif
> +}
> +
> +
> +static PyObject *
> +_reflink_linux_file_copy(PyObject *self, PyObject *args)
> +{
> +    int eintr_retry, error, fd_in, fd_out;
> +    off_t offset_out;
> +    ssize_t copyfunc_ret;
> +#ifdef SEEK_DATA
> +    /* Linux 3.1 and later support SEEK_DATA (for sparse file support).
> +     * This code uses copy_file_range if possible, and falls back to
> +     * sendfile for cross-device or when the copy_file_range syscall
> +     * is not available (less than Linux 4.5). This will fail for
> +     * Linux less than 3.1, which does not support the lseek SEEK_DATA
> +     * parameter. The caller should perform a runtime test to verify
> +     * that this function works with the running kernel.
> +     */
> +    off_t offset_data, offset_hole;
> +    ssize_t (*copyfunc)(int, int, loff_t *, size_t);
> +
> +    if (!PyArg_ParseTuple(args, "ii", &fd_in, &fd_out))
> +        return NULL;
> +
> +    eintr_retry = 1;
> +    offset_out = 0;
> +    offset_data = 0;
> +    copyfunc = cfr_wrapper;
> +
> +    while (eintr_retry) {
> +
> +        error = 0;
> +
> +        Py_BEGIN_ALLOW_THREADS
> +
> +        while (1) {
> +            /* Use lseek SEEK_DATA/SEEK_HOLE for sparse file support,
> +             * as suggested in the copy_file_range man page.
> +             */
> +            offset_data = lseek(fd_in, offset_out, SEEK_DATA);
> +            if (offset_data < 0) {
> +                if (errno == ENXIO) {
> +                    /* EOF */
> +                    break;
> +                }
> +                error = 1;
> +                break;
> +            }
> +
> +            /* Create sparse empty blocks in the output file, up
> +             * until the next location that will contain data.
> +             */
> +            if (offset_data > offset_out) {
> +                offset_out = lseek(fd_out, offset_data, SEEK_SET);
> +                if (offset_out < 0) {
> +                    error = 1;
> +                    break;
> +                }
> +            }
> +
> +            /* Locate the next hole, so that we know when to
> +             * stop copying. There is an implicit hole at the
> +             * end of the file.
> +             */
> +            offset_hole = lseek(fd_in, offset_data, SEEK_HOLE);
> +            if (offset_hole < 0) {
> +                error = 1;
> +                break;
> +            }
> +
> +            /* Revert SEEK_HOLE offset change, since we're going
> +             * to copy the data that comes before the hole.
> +             */
> +            if (lseek(fd_in, offset_out, SEEK_SET) < 0) {
> +                error = 1;
> +                break;
> +            }
> +
> +            copyfunc_ret = copyfunc(fd_out,
> +                                    fd_in,
> +                                    &offset_out,
> +                                    offset_hole - offset_data);
> +
> +            if (copyfunc_ret < 0) {
> +                if ((errno == EXDEV || errno == ENOSYS) &&
> +                    copyfunc == cfr_wrapper) {
> +                    /* Use sendfile instead of copy_file_range for
> +                     * cross-device copies, or when the copy_file_range
> +                     * syscall is not available (less than Linux 4.5).
> +                     */
> +                    copyfunc = sendfile;
> +                    copyfunc_ret = copyfunc(fd_out,
> +                                            fd_in,
> +                                            &offset_out,
> +                                            offset_hole - offset_data);
> +
> +                    if (copyfunc_ret < 0) {

I think you still should have a proper fallback for sendfile() refusing
to do its job.

> +                        error = 1;
> +                        break;
> +                    }
> +                }
> +                else {
> +                    error = 1;
> +                    break;
> +                }
> +            }
> +        }
> +#else
> +    /* Less than Linux 3.1 does not support SEEK_DATA or copy_file_range,
> +     * so just use sendfile for in-kernel copy. This will fail for Linux
> +     * versions from 2.6.0 to 2.6.32, because sendfile does not support
> +     * writing to regular files. The caller should perform a runtime
> +     * test to verify that this function works with the running kernel.
> +     */
> +    struct stat sb;
> +    int stat_acquired;
> +
> +    if (!PyArg_ParseTuple(args, "ii", &fd_in, &fd_out))
> +        return NULL;
> +
> +    eintr_retry = 1;
> +    offset_out = 0;
> +    stat_acquired = 0;
> +
> +    while (eintr_retry) {
> +
> +        error = 0;
> +
> +        Py_BEGIN_ALLOW_THREADS
> +
> +        if (!stat_acquired && fstat(fd_in, &sb) < 0) {
> +            error = 1;
> +        }
> +        else {
> +            stat_acquired = 1;
> +            while (offset_out < sb.st_size) {
> +                copyfunc_ret = sendfile(fd_out,
> +                                        fd_in,
> +                                        &offset_out,
> +                                        sb.st_size - offset_out);
> +
> +                if (copyfunc_ret < 0) {

Likewise, especially that old kernels may refuse file-to-file copy here.

> +                    error = 1;
> +                    break;
> +                }
> +            }
> +        }
> +#endif
> +        Py_END_ALLOW_THREADS
> +
> +        if (!(error && errno == EINTR && PyErr_CheckSignals() == 0))
> +            eintr_retry = 0;
> +    }
> +
> +    if (error)
> +        return PyErr_SetFromErrno(PyExc_OSError);
> +
> +    return Py_BuildValue("i", offset_out);
> +}

-- 
Best regards,
Michał Górny