Source file src/pkg/internal/poll/splice_linux.go

     1	// Copyright 2018 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package poll
     6	
     7	import (
     8		"sync/atomic"
     9		"syscall"
    10		"unsafe"
    11	)
    12	
    13	const (
    14		// spliceNonblock makes calls to splice(2) non-blocking.
    15		spliceNonblock = 0x2
    16	
    17		// maxSpliceSize is the maximum amount of data Splice asks
    18		// the kernel to move in a single call to splice(2).
    19		maxSpliceSize = 4 << 20
    20	)
    21	
    22	// Splice transfers at most remain bytes of data from src to dst, using the
    23	// splice system call to minimize copies of data from and to userspace.
    24	//
    25	// Splice creates a temporary pipe, to serve as a buffer for the data transfer.
    26	// src and dst must both be stream-oriented sockets.
    27	//
    28	// If err != nil, sc is the system call which caused the error.
    29	func Splice(dst, src *FD, remain int64) (written int64, handled bool, sc string, err error) {
    30		prfd, pwfd, sc, err := newTempPipe()
    31		if err != nil {
    32			return 0, false, sc, err
    33		}
    34		defer destroyTempPipe(prfd, pwfd)
    35		var inPipe, n int
    36		for err == nil && remain > 0 {
    37			max := maxSpliceSize
    38			if int64(max) > remain {
    39				max = int(remain)
    40			}
    41			inPipe, err = spliceDrain(pwfd, src, max)
    42			// The operation is considered handled if splice returns no
    43			// error, or an error other than EINVAL. An EINVAL means the
    44			// kernel does not support splice for the socket type of src.
    45			// The failed syscall does not consume any data so it is safe
    46			// to fall back to a generic copy.
    47			//
    48			// spliceDrain should never return EAGAIN, so if err != nil,
    49			// Splice cannot continue.
    50			//
    51			// If inPipe == 0 && err == nil, src is at EOF, and the
    52			// transfer is complete.
    53			handled = handled || (err != syscall.EINVAL)
    54			if err != nil || (inPipe == 0 && err == nil) {
    55				break
    56			}
    57			n, err = splicePump(dst, prfd, inPipe)
    58			if n > 0 {
    59				written += int64(n)
    60				remain -= int64(n)
    61			}
    62		}
    63		if err != nil {
    64			return written, handled, "splice", err
    65		}
    66		return written, true, "", nil
    67	}
    68	
    69	// spliceDrain moves data from a socket to a pipe.
    70	//
    71	// Invariant: when entering spliceDrain, the pipe is empty. It is either in its
    72	// initial state, or splicePump has emptied it previously.
    73	//
    74	// Given this, spliceDrain can reasonably assume that the pipe is ready for
    75	// writing, so if splice returns EAGAIN, it must be because the socket is not
    76	// ready for reading.
    77	//
    78	// If spliceDrain returns (0, nil), src is at EOF.
    79	func spliceDrain(pipefd int, sock *FD, max int) (int, error) {
    80		if err := sock.readLock(); err != nil {
    81			return 0, err
    82		}
    83		defer sock.readUnlock()
    84		if err := sock.pd.prepareRead(sock.isFile); err != nil {
    85			return 0, err
    86		}
    87		for {
    88			n, err := splice(pipefd, sock.Sysfd, max, spliceNonblock)
    89			if err != syscall.EAGAIN {
    90				return n, err
    91			}
    92			if err := sock.pd.waitRead(sock.isFile); err != nil {
    93				return n, err
    94			}
    95		}
    96	}
    97	
    98	// splicePump moves all the buffered data from a pipe to a socket.
    99	//
   100	// Invariant: when entering splicePump, there are exactly inPipe
   101	// bytes of data in the pipe, from a previous call to spliceDrain.
   102	//
   103	// By analogy to the condition from spliceDrain, splicePump
   104	// only needs to poll the socket for readiness, if splice returns
   105	// EAGAIN.
   106	//
   107	// If splicePump cannot move all the data in a single call to
   108	// splice(2), it loops over the buffered data until it has written
   109	// all of it to the socket. This behavior is similar to the Write
   110	// step of an io.Copy in userspace.
   111	func splicePump(sock *FD, pipefd int, inPipe int) (int, error) {
   112		if err := sock.writeLock(); err != nil {
   113			return 0, err
   114		}
   115		defer sock.writeUnlock()
   116		if err := sock.pd.prepareWrite(sock.isFile); err != nil {
   117			return 0, err
   118		}
   119		written := 0
   120		for inPipe > 0 {
   121			n, err := splice(sock.Sysfd, pipefd, inPipe, spliceNonblock)
   122			// Here, the condition n == 0 && err == nil should never be
   123			// observed, since Splice controls the write side of the pipe.
   124			if n > 0 {
   125				inPipe -= n
   126				written += n
   127				continue
   128			}
   129			if err != syscall.EAGAIN {
   130				return written, err
   131			}
   132			if err := sock.pd.waitWrite(sock.isFile); err != nil {
   133				return written, err
   134			}
   135		}
   136		return written, nil
   137	}
   138	
   139	// splice wraps the splice system call. Since the current implementation
   140	// only uses splice on sockets and pipes, the offset arguments are unused.
   141	// splice returns int instead of int64, because callers never ask it to
   142	// move more data in a single call than can fit in an int32.
   143	func splice(out int, in int, max int, flags int) (int, error) {
   144		n, err := syscall.Splice(in, nil, out, nil, max, flags)
   145		return int(n), err
   146	}
   147	
   148	var disableSplice unsafe.Pointer
   149	
   150	// newTempPipe sets up a temporary pipe for a splice operation.
   151	func newTempPipe() (prfd, pwfd int, sc string, err error) {
   152		p := (*bool)(atomic.LoadPointer(&disableSplice))
   153		if p != nil && *p {
   154			return -1, -1, "splice", syscall.EINVAL
   155		}
   156	
   157		var fds [2]int
   158		// pipe2 was added in 2.6.27 and our minimum requirement is 2.6.23, so it
   159		// might not be implemented. Falling back to pipe is possible, but prior to
   160		// 2.6.29 splice returns -EAGAIN instead of 0 when the connection is
   161		// closed.
   162		const flags = syscall.O_CLOEXEC | syscall.O_NONBLOCK
   163		if err := syscall.Pipe2(fds[:], flags); err != nil {
   164			return -1, -1, "pipe2", err
   165		}
   166	
   167		if p == nil {
   168			p = new(bool)
   169			defer atomic.StorePointer(&disableSplice, unsafe.Pointer(p))
   170	
   171			// F_GETPIPE_SZ was added in 2.6.35, which does not have the -EAGAIN bug.
   172			if _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fds[0]), syscall.F_GETPIPE_SZ, 0); errno != 0 {
   173				*p = true
   174				destroyTempPipe(fds[0], fds[1])
   175				return -1, -1, "fcntl", errno
   176			}
   177		}
   178	
   179		return fds[0], fds[1], "", nil
   180	}
   181	
   182	// destroyTempPipe destroys a temporary pipe.
   183	func destroyTempPipe(prfd, pwfd int) error {
   184		err := CloseFunc(prfd)
   185		err1 := CloseFunc(pwfd)
   186		if err == nil {
   187			return err1
   188		}
   189		return err
   190	}
   191	

View as plain text