1// Copyright 2020 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package poll
6
7import (
8	"internal/syscall/unix"
9	"sync"
10	"syscall"
11)
12
13var isKernelVersionGE53 = sync.OnceValue(func() bool {
14	major, minor := unix.KernelVersion()
15	// copy_file_range(2) is broken in various ways on kernels older than 5.3,
16	// see https://go.dev/issue/42400 and
17	// https://man7.org/linux/man-pages/man2/copy_file_range.2.html#VERSIONS
18	return major > 5 || (major == 5 && minor >= 3)
19})
20
21const maxCopyFileRangeRound = 1 << 30
22
23// CopyFileRange copies at most remain bytes of data from src to dst, using
24// the copy_file_range system call. dst and src must refer to regular files.
25func CopyFileRange(dst, src *FD, remain int64) (written int64, handled bool, err error) {
26	if !isKernelVersionGE53() {
27		return 0, false, nil
28	}
29
30	for remain > 0 {
31		max := remain
32		if max > maxCopyFileRangeRound {
33			max = maxCopyFileRangeRound
34		}
35		n, err := copyFileRange(dst, src, int(max))
36		switch err {
37		case syscall.ENOSYS:
38			// copy_file_range(2) was introduced in Linux 4.5.
39			// Go supports Linux >= 2.6.33, so the system call
40			// may not be present.
41			//
42			// If we see ENOSYS, we have certainly not transferred
43			// any data, so we can tell the caller that we
44			// couldn't handle the transfer and let them fall
45			// back to more generic code.
46			return 0, false, nil
47		case syscall.EXDEV, syscall.EINVAL, syscall.EIO, syscall.EOPNOTSUPP, syscall.EPERM:
48			// Prior to Linux 5.3, it was not possible to
49			// copy_file_range across file systems. Similarly to
50			// the ENOSYS case above, if we see EXDEV, we have
51			// not transferred any data, and we can let the caller
52			// fall back to generic code.
53			//
54			// As for EINVAL, that is what we see if, for example,
55			// dst or src refer to a pipe rather than a regular
56			// file. This is another case where no data has been
57			// transferred, so we consider it unhandled.
58			//
59			// If src and dst are on CIFS, we can see EIO.
60			// See issue #42334.
61			//
62			// If the file is on NFS, we can see EOPNOTSUPP.
63			// See issue #40731.
64			//
65			// If the process is running inside a Docker container,
66			// we might see EPERM instead of ENOSYS. See issue
67			// #40893. Since EPERM might also be a legitimate error,
68			// don't mark copy_file_range(2) as unsupported.
69			return 0, false, nil
70		case nil:
71			if n == 0 {
72				// If we did not read any bytes at all,
73				// then this file may be in a file system
74				// where copy_file_range silently fails.
75				// https://lore.kernel.org/linux-fsdevel/[email protected]/T/#m05753578c7f7882f6e9ffe01f981bc223edef2b0
76				if written == 0 {
77					return 0, false, nil
78				}
79				// Otherwise src is at EOF, which means
80				// we are done.
81				return written, true, nil
82			}
83			remain -= n
84			written += n
85		default:
86			return written, true, err
87		}
88	}
89	return written, true, nil
90}
91
92// copyFileRange performs one round of copy_file_range(2).
93func copyFileRange(dst, src *FD, max int) (written int64, err error) {
94	// The signature of copy_file_range(2) is:
95	//
96	// ssize_t copy_file_range(int fd_in, loff_t *off_in,
97	//                         int fd_out, loff_t *off_out,
98	//                         size_t len, unsigned int flags);
99	//
100	// Note that in the call to unix.CopyFileRange below, we use nil
101	// values for off_in and off_out. For the system call, this means
102	// "use and update the file offsets". That is why we must acquire
103	// locks for both file descriptors (and why this whole machinery is
104	// in the internal/poll package to begin with).
105	if err := dst.writeLock(); err != nil {
106		return 0, err
107	}
108	defer dst.writeUnlock()
109	if err := src.readLock(); err != nil {
110		return 0, err
111	}
112	defer src.readUnlock()
113	var n int
114	for {
115		n, err = unix.CopyFileRange(src.Sysfd, nil, dst.Sysfd, nil, max, 0)
116		if err != syscall.EINTR {
117			break
118		}
119	}
120	return int64(n), err
121}
122