diff --git a/src/internal/poll/fd_windows.go b/src/internal/poll/fd_windows.go index c624a16a2a..269bf4f9b6 100644 --- a/src/internal/poll/fd_windows.go +++ b/src/internal/poll/fd_windows.go @@ -161,7 +161,6 @@ func newWSAMsg(p []byte, oob []byte, flags int, rsa *wsaRsa) *windows.WSAMsg { // The returned object can't be allocated in the stack because it is accessed asynchronously // by Windows in between several system calls. If the stack frame is moved while that happens, // then Windows may access invalid memory. - // TODO(qmuntal): investigate using runtime.Pinner keeping this path allocation-free. // Use a pool to reuse allocations. msg := wsaMsgPool.Get().(*windows.WSAMsg) @@ -253,8 +252,13 @@ func (fd *FD) waitIO(o *operation) error { // execIO executes a single IO operation o. // It supports both synchronous and asynchronous IO. -// buf, if not nil, will be pinned during the lifetime of the operation. -func (fd *FD) execIO(mode int, submit func(o *operation) (uint32, error), buf []byte) (int, error) { +// pinPtrs is a list of pointers that will be pinned to a fixed location in memory +// during the lifetime of the operation. +func (fd *FD) execIO( + mode int, + submit func(o *operation) (uint32, error), + pinPtrs ...any, +) (int, error) { // Notify runtime netpoll about starting IO. err := fd.pd.prepare(mode, fd.isFile) if err != nil { @@ -268,21 +272,19 @@ func (fd *FD) execIO(mode int, submit func(o *operation) (uint32, error), buf [] } o.setOffset(fd.offset) if !fd.isBlocking { - if len(buf) > 0 { - ptr := unsafe.SliceData(buf) - if mode == 'r' { - fd.readPinner.Pin(ptr) - } else { - fd.writePinner.Pin(ptr) - } - defer func() { - if mode == 'r' { - fd.readPinner.Unpin() - } else { - fd.writePinner.Unpin() - } - }() + var pinner *runtime.Pinner + if mode == 'r' { + pinner = &fd.readPinner + } else { + pinner = &fd.writePinner } + defer pinner.Unpin() + + pinner.Pin(o) + for _, ptr := range pinPtrs { + pinner.Pin(ptr) + } + if !fd.associated { // If the handle is opened for overlapped IO but we can't // use the runtime poller, then we need to use an @@ -560,6 +562,13 @@ func (fd *FD) Close() error { // See golang.org/issue/26923. const maxRW = 1 << 30 // 1GB is large enough and keeps subsequent reads aligned +func pinPtrsFromBuf(buf []byte) []any { + if len(buf) == 0 { + return nil + } + return []any{unsafe.SliceData(buf)} +} + // Read implements io.Reader. func (fd *FD) Read(buf []byte) (int, error) { if fd.kind == kindFile { @@ -587,7 +596,7 @@ func (fd *FD) Read(buf []byte) (int, error) { n, err = fd.execIO('r', func(o *operation) (qty uint32, err error) { err = syscall.ReadFile(fd.Sysfd, buf, &qty, fd.overlapped(o)) return qty, err - }, buf) + }, pinPtrsFromBuf(buf)...) fd.addOffset(n) switch err { case syscall.ERROR_HANDLE_EOF: @@ -603,7 +612,7 @@ func (fd *FD) Read(buf []byte) (int, error) { var flags uint32 err = syscall.WSARecv(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &o.o, nil) return qty, err - }, buf) + }, pinPtrsFromBuf(buf)...) if race.Enabled { race.Acquire(unsafe.Pointer(&ioSync)) } @@ -721,7 +730,7 @@ func (fd *FD) Pread(buf []byte, off int64) (int, error) { err = syscall.ReadFile(fd.Sysfd, buf, &qty, &o.o) return qty, err - }, buf) + }, pinPtrsFromBuf(buf)...) if err == syscall.ERROR_HANDLE_EOF { err = io.EOF } @@ -750,7 +759,7 @@ func (fd *FD) ReadFrom(buf []byte) (int, syscall.Sockaddr, error) { var flags uint32 err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &rsa.name, &rsa.namelen, &o.o, nil) return qty, err - }, buf) + }, unsafe.SliceData(buf), rsa) err = fd.eofError(n, err) if err != nil { return n, nil, err @@ -778,7 +787,7 @@ func (fd *FD) ReadFromInet4(buf []byte, sa4 *syscall.SockaddrInet4) (int, error) var flags uint32 err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &rsa.name, &rsa.namelen, &o.o, nil) return qty, err - }, buf) + }, unsafe.SliceData(buf), rsa) err = fd.eofError(n, err) if err != nil { return n, err @@ -806,7 +815,7 @@ func (fd *FD) ReadFromInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error) var flags uint32 err = syscall.WSARecvFrom(fd.Sysfd, newWsaBuf(buf), 1, &qty, &flags, &rsa.name, &rsa.namelen, &o.o, nil) return qty, err - }, buf) + }, unsafe.SliceData(buf), rsa) err = fd.eofError(n, err) if err != nil { return n, err @@ -845,7 +854,7 @@ func (fd *FD) Write(buf []byte) (int, error) { n, err = fd.execIO('w', func(o *operation) (qty uint32, err error) { err = syscall.WriteFile(fd.Sysfd, b, &qty, fd.overlapped(o)) return qty, err - }, b) + }, pinPtrsFromBuf(b)...) fd.addOffset(n) case kindNet: if race.Enabled { @@ -854,7 +863,7 @@ func (fd *FD) Write(buf []byte) (int, error) { n, err = fd.execIO('w', func(o *operation) (qty uint32, err error) { err = syscall.WSASend(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, &o.o, nil) return qty, err - }, b) + }, pinPtrsFromBuf(b)...) } ntotal += n if ntotal == len(buf) || err != nil { @@ -927,6 +936,7 @@ func (fd *FD) Pwrite(buf []byte, off int64) (int, error) { if max-ntotal > maxRW { max = ntotal + maxRW } + b := buf[ntotal:max] n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { // Overlapped handles don't have the file pointer updated // when performing I/O operations, so there is no need to @@ -942,9 +952,9 @@ func (fd *FD) Pwrite(buf []byte, off int64) (int, error) { } o.setOffset(off + int64(ntotal)) - err = syscall.WriteFile(fd.Sysfd, buf[ntotal:max], &qty, &o.o) + err = syscall.WriteFile(fd.Sysfd, b, &qty, &o.o) return qty, err - }, buf[ntotal:max]) + }, pinPtrsFromBuf(b)...) if n > 0 { ntotal += n } @@ -974,7 +984,7 @@ func (fd *FD) Writev(buf *[][]byte) (int64, error) { n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = syscall.WSASend(fd.Sysfd, &(*bufs)[0], uint32(len(*bufs)), &qty, 0, &o.o, nil) return qty, err - }, nil) + }) TestHookDidWritev(n) consume(buf, int64(n)) return int64(n), err @@ -992,7 +1002,7 @@ func (fd *FD) WriteTo(buf []byte, sa syscall.Sockaddr) (int, error) { n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = syscall.WSASendto(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa, &o.o, nil) return qty, err - }, nil) + }) return n, err } @@ -1005,7 +1015,7 @@ func (fd *FD) WriteTo(buf []byte, sa syscall.Sockaddr) (int, error) { n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = syscall.WSASendto(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa, &o.o, nil) return qty, err - }, b) + }, unsafe.SliceData(b)) ntotal += int(n) if err != nil { return ntotal, err @@ -1027,7 +1037,7 @@ func (fd *FD) WriteToInet4(buf []byte, sa4 *syscall.SockaddrInet4) (int, error) n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = windows.WSASendtoInet4(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa4, &o.o, nil) return qty, err - }, nil) + }) return n, err } @@ -1040,7 +1050,7 @@ func (fd *FD) WriteToInet4(buf []byte, sa4 *syscall.SockaddrInet4) (int, error) n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = windows.WSASendtoInet4(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa4, &o.o, nil) return qty, err - }, b) + }, unsafe.SliceData(b)) ntotal += int(n) if err != nil { return ntotal, err @@ -1062,7 +1072,7 @@ func (fd *FD) WriteToInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error) n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = windows.WSASendtoInet6(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, 0, sa6, &o.o, nil) return qty, err - }, nil) + }) return n, err } @@ -1075,7 +1085,7 @@ func (fd *FD) WriteToInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error) n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = windows.WSASendtoInet6(fd.Sysfd, newWsaBuf(b), 1, &qty, 0, sa6, &o.o, nil) return qty, err - }, b) + }, unsafe.SliceData(b)) ntotal += int(n) if err != nil { return ntotal, err @@ -1091,7 +1101,7 @@ func (fd *FD) WriteToInet6(buf []byte, sa6 *syscall.SockaddrInet6) (int, error) func (fd *FD) ConnectEx(ra syscall.Sockaddr) error { _, err := fd.execIO('w', func(o *operation) (uint32, error) { return 0, ConnectExFunc(fd.Sysfd, ra, nil, 0, nil, &o.o) - }, nil) + }) return err } @@ -1102,7 +1112,7 @@ func (fd *FD) acceptOne(s syscall.Handle, rawsa []syscall.RawSockaddrAny) (strin err = AcceptFunc(fd.Sysfd, s, (*byte)(unsafe.Pointer(&rawsa[0])), 0, rsan, rsan, &qty, &o.o) return qty, err - }, nil) + }) if err != nil { CloseFunc(s) return "acceptex", err @@ -1268,7 +1278,7 @@ func (fd *FD) RawRead(f func(uintptr) bool) error { } err = syscall.WSARecv(fd.Sysfd, &syscall.WSABuf{}, 1, &qty, &flags, &o.o, nil) return qty, err - }, nil) + }) if err == windows.WSAEMSGSIZE { // expected with a 0-byte peek, ignore. } else if err != nil { @@ -1361,7 +1371,7 @@ func (fd *FD) ReadMsg(p []byte, oob []byte, flags int) (int, int, int, syscall.S n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil) return qty, err - }, nil) + }, rsa, msg) err = fd.eofError(n, err) var sa syscall.Sockaddr if err == nil { @@ -1388,7 +1398,7 @@ func (fd *FD) ReadMsgInet4(p []byte, oob []byte, flags int, sa4 *syscall.Sockadd n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil) return qty, err - }, nil) + }, rsa, msg) err = fd.eofError(n, err) if err == nil { rawToSockaddrInet4(msg.Name, sa4) @@ -1414,7 +1424,7 @@ func (fd *FD) ReadMsgInet6(p []byte, oob []byte, flags int, sa6 *syscall.Sockadd n, err := fd.execIO('r', func(o *operation) (qty uint32, err error) { err = windows.WSARecvMsg(fd.Sysfd, msg, &qty, &o.o, nil) return qty, err - }, nil) + }, rsa, msg) err = fd.eofError(n, err) if err == nil { rawToSockaddrInet6(msg.Name, sa6) @@ -1448,7 +1458,7 @@ func (fd *FD) WriteMsg(p []byte, oob []byte, sa syscall.Sockaddr) (int, int, err n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil) return qty, err - }, nil) + }, rsa, msg) return n, int(msg.Control.Len), err } @@ -1474,7 +1484,7 @@ func (fd *FD) WriteMsgInet4(p []byte, oob []byte, sa *syscall.SockaddrInet4) (in n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil) return qty, err - }, nil) + }, rsa, msg) return n, int(msg.Control.Len), err } @@ -1500,7 +1510,7 @@ func (fd *FD) WriteMsgInet6(p []byte, oob []byte, sa *syscall.SockaddrInet6) (in n, err := fd.execIO('w', func(o *operation) (qty uint32, err error) { err = windows.WSASendMsg(fd.Sysfd, msg, 0, nil, &o.o, nil) return qty, err - }, nil) + }, rsa, msg) return n, int(msg.Control.Len), err } diff --git a/src/internal/poll/sendfile_windows.go b/src/internal/poll/sendfile_windows.go index 18123f8629..76c6f7612e 100644 --- a/src/internal/poll/sendfile_windows.go +++ b/src/internal/poll/sendfile_windows.go @@ -75,7 +75,7 @@ func SendFile(fd *FD, src uintptr, size int64) (written int64, err error, handle return 0, err } return uint32(chunkSize), nil - }, nil) + }) if err != nil { return written, err, written > 0 }