From 37b0f666296387a2fd21f09cb4314c01b825e6f2 Mon Sep 17 00:00:00 2001 From: wangzhuowei Date: Mon, 18 Sep 2023 19:06:00 +0800 Subject: [PATCH] perf: use block syscall enter for epoll_wait --- poll_default_linux.go | 18 ++++++++++++--- sys_epoll_linux.go | 45 ++++++++++++++++++++++++++++---------- sys_epoll_linux_amd64.go | 30 +++++++++++++++++++++++++ sys_epoll_linux_arm64.go | 39 ++++----------------------------- sys_epoll_linux_loong64.go | 40 +++------------------------------ 5 files changed, 85 insertions(+), 87 deletions(-) create mode 100644 sys_epoll_linux_amd64.go diff --git a/poll_default_linux.go b/poll_default_linux.go index a0087ee0..fa6a5683 100644 --- a/poll_default_linux.go +++ b/poll_default_linux.go @@ -97,13 +97,25 @@ func (p *defaultPoll) Wait() (err error) { if n == p.size && p.size < 128*1024 { p.Reset(p.size<<1, caps) } - n, err = EpollWait(p.fd, p.events, msec) + // msec: 0(raw) => 1ms(sched,raw) => -1(block_syscall) + // poller's G will hold P at most 1ms + if msec > 0 { + n, err = EpollWaitRaw(p.fd, p.events, msec) + } else if msec == 0 { + n, err = EpollWaitRaw(p.fd, p.events, msec) + } else { // < 0 + n, err = EpollWaitBlock(p.fd, p.events, msec) + } if err != nil && err != syscall.EINTR { return err } if n <= 0 { - msec = -1 - runtime.Gosched() + if msec > 0 { + msec = -1 // no need to sched, because we will use block syscall + } else if msec == 0 { + msec = 1 + runtime.Gosched() + } continue } msec = 0 diff --git a/sys_epoll_linux.go b/sys_epoll_linux.go index 528c34ed..7bf18574 100644 --- a/sys_epoll_linux.go +++ b/sys_epoll_linux.go @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !arm64 && !loong64 -// +build !arm64,!loong64 +//go:build linux +// +build linux package netpoll @@ -22,12 +22,11 @@ import ( "unsafe" ) -const EPOLLET = -syscall.EPOLLET +//go:linkname entersyscallblock runtime.entersyscallblock +func entersyscallblock() -type epollevent struct { - events uint32 - data [8]byte // unaligned uintptr -} +//go:linkname exitsyscall runtime.exitsyscall +func exitsyscall() // EpollCreate implements epoll_create1. func EpollCreate(flag int) (fd int, err error) { @@ -51,14 +50,36 @@ func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { // EpollWait implements epoll_wait. func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_WAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) + r0, _, err = syscall.Syscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0) + if err == syscall.Errno(0) { + err = nil } + return int(r0), err +} + +func EpollWaitRaw(epfd int, events []epollevent, msec int) (n int, err error) { + var r0 uintptr + r0, _, err = syscall.RawSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0) if err == syscall.Errno(0) { err = nil } return int(r0), err } + +func EpollWaitBlock(epfd int, events []epollevent, msec int) (n int, err error) { + r0, _, errno := BlockSyscall6(SYS_EPOLL_WAIT, uintptr(epfd), uintptr(unsafe.Pointer(&events[0])), uintptr(len(events)), uintptr(msec), 0, 0) + if errno == syscall.Errno(0) { + err = nil + } else { + err = errno + } + return int(r0), err +} + +//go:nosplit +func BlockSyscall6(trap, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) { + entersyscallblock() + r1, r2, err = syscall.RawSyscall6(trap, a1, a2, a3, a4, a5, a6) + exitsyscall() + return r1, r2, err +} diff --git a/sys_epoll_linux_amd64.go b/sys_epoll_linux_amd64.go new file mode 100644 index 00000000..f05e8bdc --- /dev/null +++ b/sys_epoll_linux_amd64.go @@ -0,0 +1,30 @@ +// Copyright 2022 CloudWeGo Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !arm64 && !loong64 +// +build !arm64,!loong64 + +package netpoll + +import ( + "syscall" +) + +const EPOLLET = -syscall.EPOLLET +const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_WAIT + +type epollevent struct { + events uint32 + data [8]byte // unaligned uintptr +} diff --git a/sys_epoll_linux_arm64.go b/sys_epoll_linux_arm64.go index e8d6094d..6628e56f 100644 --- a/sys_epoll_linux_arm64.go +++ b/sys_epoll_linux_arm64.go @@ -12,51 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +//go:build arm64 +// +build arm64 + package netpoll import ( "syscall" - "unsafe" ) const EPOLLET = syscall.EPOLLET +const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_PWAIT type epollevent struct { events uint32 _ int32 data [8]byte // unaligned uintptr } - -// EpollCreate implements epoll_create1. -func EpollCreate(flag int) (fd int, err error) { - var r0 uintptr - r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} - -// EpollCtl implements epoll_ctl. -func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { - _, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return err -} - -// EpollWait implements epoll_wait. -func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { - var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) - } - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} diff --git a/sys_epoll_linux_loong64.go b/sys_epoll_linux_loong64.go index ecf36c13..6e761d55 100644 --- a/sys_epoll_linux_loong64.go +++ b/sys_epoll_linux_loong64.go @@ -12,54 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build linux && loong64 -// +build linux,loong64 +//go:build loong64 +// +build loong64 package netpoll import ( "syscall" - "unsafe" ) const EPOLLET = syscall.EPOLLET +const SYS_EPOLL_WAIT = syscall.SYS_EPOLL_PWAIT type epollevent struct { events uint32 _ int32 data [8]byte // unaligned uintptr } - -// EpollCreate implements epoll_create1. -func EpollCreate(flag int) (fd int, err error) { - var r0 uintptr - r0, _, err = syscall.RawSyscall(syscall.SYS_EPOLL_CREATE1, uintptr(flag), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -} - -// EpollCtl implements epoll_ctl. -func EpollCtl(epfd int, op int, fd int, event *epollevent) (err error) { - _, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_CTL, uintptr(epfd), uintptr(op), uintptr(fd), uintptr(unsafe.Pointer(event)), 0, 0) - if err == syscall.Errno(0) { - err = nil - } - return err -} - -// EpollWait implements epoll_wait. -func EpollWait(epfd int, events []epollevent, msec int) (n int, err error) { - var r0 uintptr - var _p0 = unsafe.Pointer(&events[0]) - if msec == 0 { - r0, _, err = syscall.RawSyscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), 0, 0, 0) - } else { - r0, _, err = syscall.Syscall6(syscall.SYS_EPOLL_PWAIT, uintptr(epfd), uintptr(_p0), uintptr(len(events)), uintptr(msec), 0, 0) - } - if err == syscall.Errno(0) { - err = nil - } - return int(r0), err -}