internal/poll: pin all objects needed during overlapped I/O

Windows requires these objects to remain valid during the whole
overlapped operation. This currently works because we have a non-moving
GC and these objects are taken from a sync.Pool.

For correctness, and to ensure a possible moving GC in the future does
not break it in a similar manner, pin these objects in execIO.

Updates #77975.

Change-Id: Iff07009d40e4a439026a961a6dca9f6843cbd61d
Reviewed-on: https://go-review.googlesource.com/c/go/+/753560
Auto-Submit: Damien Neil <dneil@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Quim Muntal <quimmuntal@gmail.com>
Reviewed-by: Damien Neil <dneil@google.com>
Reviewed-by: Carlos Amedee <carlos@golang.org>

src/internal/poll/fd_windows.go

@@ -161,7 +161,6 @@ func newWSAMsg(p []byte, oob []byte, flags int, rsa *wsaRsa) *windows.WSAMsg {
 	// The returned object can't be allocated in the stack because it is accessed asynchronously
 	// by Windows in between several system calls. If the stack frame is moved while that happens,
 	// then Windows may access invalid memory.
-	// TODO(qmuntal): investigate using runtime.Pinner keeping this path allocation-free.
 
 	// Use a pool to reuse allocations.
 	msg := wsaMsgPool.Get().(*windows.WSAMsg)
@@ -253,8 +252,13 @@ func (fd *FD) waitIO(o *operation) error {
 
 // execIO executes a single IO operation o.
 // It supports both synchronous and asynchronous IO.
-// buf, if not nil, will be pinned during the lifetime of the operation.
-func (fd *FD) execIO(mode int, submit func(o *operation) (uint32, error), buf []byte) (int, error) {
+// pinPtrs is a list of pointers that will be pinned to a fixed location in memory
+// during the lifetime of the operation.
+func (fd *FD) execIO(
+	mode int,
+	submit func(o *operation) (uint32, error),
+	pinPtrs ...any,
+) (int, error) {
 	// Notify runtime netpoll about starting IO.
 	err := fd.pd.prepare(mode, fd.isFile)
 	if err != nil {
@@ -268,21 +272,19 @@ func (fd *FD) execIO(mode int, submit func(o *operation) (uint32, error), buf []
 	}
 	o.setOffset(fd.offset)
 	if !fd.isBlocking {
-		if len(buf) > 0 {
-			ptr := unsafe.SliceData(buf)
-			if mode == 'r' {
-				fd.readPinner.Pin(ptr)
-			} else {
-				fd.writePinner.Pin(ptr)
-			}
-			defer func() {
-				if mode == 'r' {
-					fd.readPinner.Unpin()
-				} else {
-					fd.writePinner.Unpin()
-				}
-			}()
+		var pinner *runtime.Pinner
+		if mode == 'r' {
+			pinner = &fd.readPinner
+		} else {
+			pinner = &fd.writePinner
 		}
+		defer pinner.Unpin()
+
+		pinner.Pin(o)
+		for _, ptr := range pinPtrs {
+			pinner.Pin(ptr)
+		}
+
 		if !fd.associated {
 			// If the handle is opened for overlapped IO but we can't
 			// use the runtime poller, then we need to use an
@@ -560,6 +562,13 @@ func (fd *FD) Close() error {
 // See golang.org/issue/26923.
 const maxRW = 1 << 30 // 1GB is large enough and keeps subsequent reads aligned
 
+func pinPtrsFromBuf(buf []byte) []any {
+	if len(buf) == 0 {
+		return nil
+	}
+	return []any{unsafe.SliceData(buf)}
+}
+
 // Read implements io.Reader.
 func (fd *FD) Read(buf []byte) (int, error) {
 	if fd.kind == kindFile {
@@ -587,7 +596,7 @@ func (fd *FD) Read(buf []byte) (int, error) {
 		n, err = fd.execIO('r', func(o *operation) (qty uint32, err error) {
 			err = syscall.ReadFile(fd.Sysfd, buf, &qty, fd.overlapped(o))
 			return qty, err
-		}, buf)
+		}, pinPtrsFromBuf(buf)...)
 		fd.addOffset(n)
 		switch err {
 		case syscall.ERROR_HANDLE_EOF:
@@ -603,7 +612,7 @@ func (fd *FD) Read(buf []byte) (int, error) {
 			var flags uint32
 			err = syscall.WSARecv(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &o.o, nil)
 			return qty, err
-		}, buf)
+		}, pinPtrsFromBuf(buf)...)
 		if race.Enabled {
 			race.Acquire(unsafe.Pointer(&ioSync))
 		}
@@ -721,7 +730,7 @@ func (fd *FD) Pread(buf []byte, off int64) (int, error) {
 
 		err = syscall.ReadFile(fd.Sysfd, buf, &qty, &o.o)
 		return qty, err
-	}, buf)
+	}, pinPtrsFromBuf(buf)...)
 	if err == syscall.ERROR_HANDLE_EOF {
 		err = io.EOF
 	}
@@ -750,7 +759,7 @@ func (fd *FD) ReadFrom(buf []byte) (int, syscall.Sockaddr, error) {
 		var flags uint32
 		err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &rsa.name, &rsa.namelen, &o.o, nil)
 		return qty, err
-	}, buf)
+	}, unsafe.SliceData(buf), rsa)
 	err = fd.eofError(n, err)
 	if err != nil {
 		return n, nil, err
@@ -778,7 +787,7 @@ func (fd *FD) ReadFromInet4(buf []byte, sa4 *syscall.SockaddrInet4) (int, error)
 		var flags uint32
 		err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &rsa.name, &rsa.namelen, &o.o, nil)
 		return qty, err
-	}, buf)
+	}, unsafe.SliceData(buf), rsa)
 	err = fd.eofError(n, err)
 	if err != nil {
 		return n, err
@@ -806,7 +815,7 @@ func (fd *FD) ReadFromInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error)
 		var flags uint32
 		err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &rsa.name, &rsa.namelen, &o.o, nil)
 		return qty, err
-	}, buf)
+	}, unsafe.SliceData(buf), rsa)
 	err = fd.eofError(n, err)
 	if err != nil {
 		return n, err
@@ -845,7 +854,7 @@ func (fd *FD) Write(buf []byte) (int, error) {
 			n, err = fd.execIO('w', func(o *operation) (qty uint32, err error) {
 				err = syscall.WriteFile(fd.Sysfd, b, &qty, fd.overlapped(o))
 				return qty, err
-			}, b)
+			}, pinPtrsFromBuf(b)...)
 			fd.addOffset(n)
 		case kindNet:
 			if race.Enabled {
@@ -854,7 +863,7 @@ func (fd *FD) Write(buf []byte) (int, error) {
 			n, err = fd.execIO('w', func(o *operation) (qty uint32, err error) {
 				err = syscall.WSASend(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, &o.o, nil)
 				return qty, err
-			}, b)
+			}, pinPtrsFromBuf(b)...)
 		}
 		ntotal += n
 		if ntotal == len(buf) || err != nil {
@@ -927,6 +936,7 @@ func (fd *FD) Pwrite(buf []byte, off int64) (int, error) {
 		if max-ntotal > maxRW {
 			max = ntotal + maxRW
 		}
+		b := buf[ntotal:max]
 		n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 			// Overlapped handles don't have the file pointer updated
 			// when performing I/O operations, so there is no need to
@@ -942,9 +952,9 @@ func (fd *FD) Pwrite(buf []byte, off int64) (int, error) {
 			}
 			o.setOffset(off + int64(ntotal))
 
-			err = syscall.WriteFile(fd.Sysfd, buf[ntotal:max], &qty, &o.o)
+			err = syscall.WriteFile(fd.Sysfd, b, &qty, &o.o)
 			return qty, err
-		}, buf[ntotal:max])
+		}, pinPtrsFromBuf(b)...)
 		if n > 0 {
 			ntotal += n
 		}
@@ -974,7 +984,7 @@ func (fd *FD) Writev(buf *[][]byte) (int64, error) {
 	n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 		err = syscall.WSASend(fd.Sysfd, &(*bufs)[0], uint32(len(*bufs)), &qty, 0, &o.o, nil)
 		return qty, err
-	}, nil)
+	})
 	TestHookDidWritev(n)
 	consume(buf, int64(n))
 	return int64(n), err
@@ -992,7 +1002,7 @@ func (fd *FD) WriteTo(buf []byte, sa syscall.Sockaddr) (int, error) {
 		n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 			err = syscall.WSASendto(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa, &o.o, nil)
 			return qty, err
-		}, nil)
+		})
 		return n, err
 	}
 
@@ -1005,7 +1015,7 @@ func (fd *FD) WriteTo(buf []byte, sa syscall.Sockaddr) (int, error) {
 		n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 			err = syscall.WSASendto(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa, &o.o, nil)
 			return qty, err
-		}, b)
+		}, unsafe.SliceData(b))
 		ntotal += int(n)
 		if err != nil {
 			return ntotal, err
@@ -1027,7 +1037,7 @@ func (fd *FD) WriteToInet4(buf []byte, sa4 *syscall.SockaddrInet4) (int, error)
 		n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 			err = windows.WSASendtoInet4(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa4, &o.o, nil)
 			return qty, err
-		}, nil)
+		})
 		return n, err
 	}
 
@@ -1040,7 +1050,7 @@ func (fd *FD) WriteToInet4(buf []byte, sa4 *syscall.SockaddrInet4) (int, error)
 		n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 			err = windows.WSASendtoInet4(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa4, &o.o, nil)
 			return qty, err
-		}, b)
+		}, unsafe.SliceData(b))
 		ntotal += int(n)
 		if err != nil {
 			return ntotal, err
@@ -1062,7 +1072,7 @@ func (fd *FD) WriteToInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error)
 		n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 			err = windows.WSASendtoInet6(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa6, &o.o, nil)
 			return qty, err
-		}, nil)
+		})
 		return n, err
 	}
 
@@ -1075,7 +1085,7 @@ func (fd *FD) WriteToInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error)
 		n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 			err = windows.WSASendtoInet6(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa6, &o.o, nil)
 			return qty, err
-		}, b)
+		}, unsafe.SliceData(b))
 		ntotal += int(n)
 		if err != nil {
 			return ntotal, err
@@ -1091,7 +1101,7 @@ func (fd *FD) WriteToInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error)
 func (fd *FD) ConnectEx(ra syscall.Sockaddr) error {
 	_, err := fd.execIO('w', func(o *operation) (uint32, error) {
 		return 0, ConnectExFunc(fd.Sysfd, ra, nil, 0, nil, &o.o)
-	}, nil)
+	})
 	return err
 }
 
@@ -1102,7 +1112,7 @@ func (fd *FD) acceptOne(s syscall.Handle, rawsa []syscall.RawSockaddrAny) (strin
 		err = AcceptFunc(fd.Sysfd, s, (*byte)(unsafe.Pointer(&rawsa[0])), 0, rsan, rsan, &qty, &o.o)
 		return qty, err
 
-	}, nil)
+	})
 	if err != nil {
 		CloseFunc(s)
 		return "acceptex", err
@@ -1268,7 +1278,7 @@ func (fd *FD) RawRead(f func(uintptr) bool) error {
 			}
 			err = syscall.WSARecv(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, &flags, &o.o, nil)
 			return qty, err
-		}, nil)
+		})
 		if err == windows.WSAEMSGSIZE {
 			// expected with a 0-byte peek, ignore.
 		} else if err != nil {
@@ -1361,7 +1371,7 @@ func (fd *FD) ReadMsg(p []byte, oob []byte, flags int) (int, int, int, syscall.S
 	n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) {
 		err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil)
 		return qty, err
-	}, nil)
+	}, rsa, msg)
 	err = fd.eofError(n, err)
 	var sa syscall.Sockaddr
 	if err == nil {
@@ -1388,7 +1398,7 @@ func (fd *FD) ReadMsgInet4(p []byte, oob []byte, flags int, sa4 *syscall.Sockadd
 	n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) {
 		err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil)
 		return qty, err
-	}, nil)
+	}, rsa, msg)
 	err = fd.eofError(n, err)
 	if err == nil {
 		rawToSockaddrInet4(msg.Name, sa4)
@@ -1414,7 +1424,7 @@ func (fd *FD) ReadMsgInet6(p []byte, oob []byte, flags int, sa6 *syscall.Sockadd
 	n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) {
 		err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil)
 		return qty, err
-	}, nil)
+	}, rsa, msg)
 	err = fd.eofError(n, err)
 	if err == nil {
 		rawToSockaddrInet6(msg.Name, sa6)
@@ -1448,7 +1458,7 @@ func (fd *FD) WriteMsg(p []byte, oob []byte, sa syscall.Sockaddr) (int, int, err
 	n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 		err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil)
 		return qty, err
-	}, nil)
+	}, rsa, msg)
 	return n, int(msg.Control.Len), err
 }
 
@@ -1474,7 +1484,7 @@ func (fd *FD) WriteMsgInet4(p []byte, oob []byte, sa *syscall.SockaddrInet4) (in
 	n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 		err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil)
 		return qty, err
-	}, nil)
+	}, rsa, msg)
 	return n, int(msg.Control.Len), err
 }
 
@@ -1500,7 +1510,7 @@ func (fd *FD) WriteMsgInet6(p []byte, oob []byte, sa *syscall.SockaddrInet6) (in
 	n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) {
 		err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil)
 		return qty, err
-	}, nil)
+	}, rsa, msg)
 	return n, int(msg.Control.Len), err
 }
 

src/internal/poll/sendfile_windows.go

@@ -75,7 +75,7 @@ func SendFile(fd *FD, src uintptr, size int64) (written int64, err error, handle
 				return 0, err
 			}
 			return uint32(chunkSize), nil
-		}, nil)
+		})
 		if err != nil {
 			return written, err, written > 0
 		}