From 84a2ff9e579b6042f083ccde89087ce65c0f16f6 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Tue, 21 Jan 2025 03:57:37 +0000 Subject: [PATCH] Opt-in busywait mode for futexes --- Makefile | 1 + .../wasm32-wasip1-threads/defined-symbols.txt | 4 + expected/wasm32-wasip1-threads/include-all.c | 1 + .../predefined-macros.txt | 1 + libc-bottom-half/headers/public/wasi/libc.h | 3 + .../musl/include/wasi/libc-busywait.h | 15 ++++ libc-top-half/musl/src/thread/__wait.c | 21 ++++-- .../src/thread/wasm32/__wasilibc_busywait.c | 75 +++++++++++++++++++ .../functional/pthread_cond_busywait.c | 13 ++++ .../functional/pthread_mutex_busywait.c | 13 ++++ .../functional/pthread_tsd_busywait.c | 13 ++++ test/src/misc/busywait.c | 32 ++++++++ 12 files changed, 187 insertions(+), 5 deletions(-) create mode 100644 libc-top-half/musl/include/wasi/libc-busywait.h create mode 100644 libc-top-half/musl/src/thread/wasm32/__wasilibc_busywait.c create mode 100644 test/src/libc-test/functional/pthread_cond_busywait.c create mode 100644 test/src/libc-test/functional/pthread_mutex_busywait.c create mode 100644 test/src/libc-test/functional/pthread_tsd_busywait.c create mode 100644 test/src/misc/busywait.c diff --git a/Makefile b/Makefile index 00dd30812..3555879dc 100644 --- a/Makefile +++ b/Makefile @@ -369,6 +369,7 @@ LIBC_TOP_HALF_MUSL_SOURCES += \ thread/sem_trywait.c \ thread/sem_wait.c \ thread/wasm32/wasi_thread_start.s \ + thread/wasm32/__wasilibc_busywait.c \ ) endif ifeq ($(THREAD_MODEL), single) diff --git a/expected/wasm32-wasip1-threads/defined-symbols.txt b/expected/wasm32-wasip1-threads/defined-symbols.txt index 2218541b4..7c074e2b8 100644 --- a/expected/wasm32-wasip1-threads/defined-symbols.txt +++ b/expected/wasm32-wasip1-threads/defined-symbols.txt @@ -362,6 +362,7 @@ __wasilibc_cwd_lock __wasilibc_cwd_unlock __wasilibc_deinitialize_environ __wasilibc_dttoif +__wasilibc_enable_futex_busywait_on_current_thread __wasilibc_ensure_environ __wasilibc_environ __wasilibc_fd_renumber @@ -369,6 +370,8 @@ __wasilibc_find_abspath __wasilibc_find_relpath __wasilibc_find_relpath_alloc __wasilibc_futex_wait +__wasilibc_futex_wait_atomic_wait +__wasilibc_futex_wait_maybe_busy __wasilibc_get_environ __wasilibc_iftodt __wasilibc_initialize_environ @@ -400,6 +403,7 @@ __wasilibc_rmdirat __wasilibc_stat __wasilibc_tell __wasilibc_unlinkat +__wasilibc_use_busy_futex __wasilibc_utimens __wasm_call_dtors __wcscoll_l diff --git a/expected/wasm32-wasip1-threads/include-all.c b/expected/wasm32-wasip1-threads/include-all.c index 3fb02ee77..3f6ad937b 100644 --- a/expected/wasm32-wasip1-threads/include-all.c +++ b/expected/wasm32-wasip1-threads/include-all.c @@ -167,6 +167,7 @@ #include #include #include +#include #include #include #include diff --git a/expected/wasm32-wasip1-threads/predefined-macros.txt b/expected/wasm32-wasip1-threads/predefined-macros.txt index 4fd890acb..96ac3b77f 100644 --- a/expected/wasm32-wasip1-threads/predefined-macros.txt +++ b/expected/wasm32-wasip1-threads/predefined-macros.txt @@ -3091,6 +3091,7 @@ #define __va_copy(d,s) __builtin_va_copy(d,s) #define __wasi__ 1 #define __wasi_api_h +#define __wasi_libc_busywait_h #define __wasi_libc_environ_h #define __wasi_libc_find_relpath_h #define __wasi_libc_h diff --git a/libc-bottom-half/headers/public/wasi/libc.h b/libc-bottom-half/headers/public/wasi/libc.h index 18d8e9e3c..bed8f9418 100644 --- a/libc-bottom-half/headers/public/wasi/libc.h +++ b/libc-bottom-half/headers/public/wasi/libc.h @@ -64,6 +64,9 @@ int __wasilibc_rename_oldat(int olddirfd, const char *oldpath, const char *newpa int __wasilibc_rename_newat(const char *oldpath, int newdirfd, const char *newpath) __attribute__((__warn_unused_result__)); +/// Enable busywait in futex on current thread. +void __wasilibc_enable_futex_busywait_on_current_thread(void); + #ifdef __cplusplus } #endif diff --git a/libc-top-half/musl/include/wasi/libc-busywait.h b/libc-top-half/musl/include/wasi/libc-busywait.h new file mode 100644 index 000000000..c90017b8e --- /dev/null +++ b/libc-top-half/musl/include/wasi/libc-busywait.h @@ -0,0 +1,15 @@ +#ifndef __wasi_libc_busywait_h +#define __wasi_libc_busywait_h + +#ifdef __cplusplus +extern "C" { +#endif + +/// Enable busywait in futex on current thread. +void __wasilibc_enable_futex_busywait_on_current_thread(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/libc-top-half/musl/src/thread/__wait.c b/libc-top-half/musl/src/thread/__wait.c index 7ffa9872d..9484c0f3d 100644 --- a/libc-top-half/musl/src/thread/__wait.c +++ b/libc-top-half/musl/src/thread/__wait.c @@ -4,6 +4,9 @@ #endif #ifndef __wasilibc_unmodified_upstream + +weak int __wasilibc_futex_wait_maybe_busy(volatile void *addr, int op, int val, int64_t max_wait_ns); + // Use WebAssembly's `wait` instruction to implement a futex. Note that `op` is // unused but retained as a parameter to match the original signature of the // syscall and that, for `max_wait_ns`, -1 (or any negative number) means wait @@ -11,12 +14,8 @@ // // Adapted from Emscripten: see // https://github.com/emscripten-core/emscripten/blob/058a9fff/system/lib/pthread/emscripten_futex_wait.c#L111-L150. -int __wasilibc_futex_wait(volatile void *addr, int op, int val, int64_t max_wait_ns) +int __wasilibc_futex_wait_atomic_wait(volatile void *addr, int op, int val, int64_t max_wait_ns) { - if ((((intptr_t)addr) & 3) != 0) { - return -EINVAL; - } - int ret = __builtin_wasm_memory_atomic_wait32((int *)addr, val, max_wait_ns); // memory.atomic.wait32 returns: @@ -32,6 +31,18 @@ int __wasilibc_futex_wait(volatile void *addr, int op, int val, int64_t max_wait assert(ret == 0); return 0; } + +int __wasilibc_futex_wait(volatile void *addr, int op, int val, int64_t max_wait_ns) +{ + if ((((intptr_t)addr) & 3) != 0) { + return -EINVAL; + } + + if (__wasilibc_futex_wait_maybe_busy) { + return __wasilibc_futex_wait_maybe_busy(addr, op, val, max_wait_ns); + } + return __wasilibc_futex_wait_atomic_wait(addr, op, val, max_wait_ns); +} #endif void __wait(volatile int *addr, volatile int *waiters, int val, int priv) diff --git a/libc-top-half/musl/src/thread/wasm32/__wasilibc_busywait.c b/libc-top-half/musl/src/thread/wasm32/__wasilibc_busywait.c new file mode 100644 index 000000000..389dc768f --- /dev/null +++ b/libc-top-half/musl/src/thread/wasm32/__wasilibc_busywait.c @@ -0,0 +1,75 @@ +#include +#include +#include + +#define DEFINE_GLOBAL_GETTER(name, core_type, c_type) \ + static inline c_type name##_get(void) { \ + c_type val; \ + __asm__( \ + ".globaltype " #name ", " #core_type "\n" \ + "global.get " #name "\n" \ + "local.set %0\n" \ + : "=r"(val)); \ + return val; \ + } +#define DEFINE_GLOBAL_SETTER(name, core_type, c_type) \ + static inline void name##_set(c_type val) { \ + __asm__( \ + ".globaltype " #name ", " #core_type "\n" \ + "local.get %0\n" \ + "global.set " #name "\n" \ + : : "r"(val)); \ + } + +#define DEFINE_RW_GLOBAL(name, core_type, c_type) \ + __asm__( \ + ".globaltype " #name ", " #core_type "\n" \ + ".global " #name "\n" \ + #name ":\n" \ + ); \ + DEFINE_GLOBAL_GETTER(name, core_type, c_type) \ + DEFINE_GLOBAL_SETTER(name, core_type, c_type) + +DEFINE_RW_GLOBAL(__wasilibc_use_busy_futex, i32, int32_t) + +void __wasilibc_enable_futex_busywait_on_current_thread(void) +{ + __wasilibc_use_busy_futex_set(1); +} + +int __wasilibc_futex_wait_atomic_wait(volatile void *addr, int op, int val, int64_t max_wait_ns); + +int __wasilibc_futex_wait_maybe_busy(volatile void *addr, int op, int val, int64_t max_wait_ns) +{ + // PLEASE NOTE THAT WE CANNOT CALL LIBC FUNCTIONS THAT USE FUTEXES HERE + + if (!__wasilibc_use_busy_futex_get()) { + return __wasilibc_futex_wait_atomic_wait(addr, op, val, max_wait_ns); + } + + struct timespec start; + int r = clock_gettime(CLOCK_REALTIME, &start); + + // If we can't get the current time, we can't wait with a timeout. + if (r) return r; + + while (1) { + // Check timeout if it's a positive value + if (max_wait_ns >= 0) { + struct timespec now; + r = clock_gettime(CLOCK_REALTIME, &now); + if (r) return r; + + int64_t elapsed_ns = (now.tv_sec - start.tv_sec) * 1000000000 + now.tv_nsec - start.tv_nsec; + if (elapsed_ns >= max_wait_ns) { + return -ETIMEDOUT; + } + } + + if (__c11_atomic_load((_Atomic int *)addr, __ATOMIC_SEQ_CST) != val) { + break; + } + } + + return 0; +} diff --git a/test/src/libc-test/functional/pthread_cond_busywait.c b/test/src/libc-test/functional/pthread_cond_busywait.c new file mode 100644 index 000000000..dd2169119 --- /dev/null +++ b/test/src/libc-test/functional/pthread_cond_busywait.c @@ -0,0 +1,13 @@ +//! filter.py(TARGET_TRIPLE): wasm32-wasip1-threads +//! add-flags.py(CFLAGS): -I. +//! add-flags.py(LDFLAGS): -Wl,--import-memory,--export-memory,--shared-memory,--max-memory=1073741824 +//! add-flags.py(RUN): --wasi threads +#include "build/download/libc-test/src/functional/pthread_cond.c" + +#include + +__attribute__((constructor)) +void __wasilibc_enable_busywait(void) +{ + __wasilibc_enable_futex_busywait_on_current_thread(); +} diff --git a/test/src/libc-test/functional/pthread_mutex_busywait.c b/test/src/libc-test/functional/pthread_mutex_busywait.c new file mode 100644 index 000000000..98b355ef8 --- /dev/null +++ b/test/src/libc-test/functional/pthread_mutex_busywait.c @@ -0,0 +1,13 @@ +//! filter.py(TARGET_TRIPLE): wasm32-wasip1-threads +//! add-flags.py(CFLAGS): -I. +//! add-flags.py(LDFLAGS): -Wl,--import-memory,--export-memory,--shared-memory,--max-memory=1073741824 +//! add-flags.py(RUN): --wasi threads +#include "build/download/libc-test/src/functional/pthread_mutex.c" + +#include + +__attribute__((constructor)) +void __wasilibc_enable_busywait(void) +{ + __wasilibc_enable_futex_busywait_on_current_thread(); +} diff --git a/test/src/libc-test/functional/pthread_tsd_busywait.c b/test/src/libc-test/functional/pthread_tsd_busywait.c new file mode 100644 index 000000000..364b8ce0c --- /dev/null +++ b/test/src/libc-test/functional/pthread_tsd_busywait.c @@ -0,0 +1,13 @@ +//! filter.py(TARGET_TRIPLE): wasm32-wasip1-threads +//! add-flags.py(CFLAGS): -I. +//! add-flags.py(LDFLAGS): -Wl,--import-memory,--export-memory,--shared-memory,--max-memory=1073741824 +//! add-flags.py(RUN): --wasi threads +#include "build/download/libc-test/src/functional/pthread_tsd.c" + +#include + +__attribute__((constructor)) +void __wasilibc_enable_busywait(void) +{ + __wasilibc_enable_futex_busywait_on_current_thread(); +} diff --git a/test/src/misc/busywait.c b/test/src/misc/busywait.c new file mode 100644 index 000000000..3644398e6 --- /dev/null +++ b/test/src/misc/busywait.c @@ -0,0 +1,32 @@ +//! filter.py(TARGET_TRIPLE): wasm32-wasip1-threads +//! add-flags.py(CFLAGS): -pthread +//! add-flags.py(LDFLAGS): -pthread +//! add-flags.py(RUN): --wasi threads + +#include +#include +#include +#include +#include + +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + +int main() { + struct timespec ts; + int ret; + + clock_gettime(CLOCK_REALTIME, &ts); + ts.tv_sec += 1; + + __wasilibc_enable_futex_busywait_on_current_thread(); + + pthread_mutex_lock(&mutex); + + ret = pthread_cond_timedwait(&cond, &mutex, &ts); + + assert(ret == ETIMEDOUT); + + pthread_mutex_unlock(&mutex); + return 0; +}