On Mon, Dec 03, 2018 at 09:14:55AM +0200, Timo Myyrä wrote:
> Hi,
>
> Here's an update to SBCL to bring it to latest version as I noticed the
> release
> fixes the thread support on OpenBSD.
The recent release does indeed fix one threading problem, but
unfortunately the sb-concurrency tests still hang for me on a
quad-core machine.
It is now possible to build the threads flavor on powerpc, but I'm not
convinced that it's worth spending time on.
> This also changes the thread backend to futex and adds support for core
> compression using zlib.
Using futexes is probably a good idea, however I'd prefer to wait
until any known issues with the portable pthreads backend are resolved
first.
> I haven't tested this that much yet but tests seem to pass on amd64 at
> least with it.
As Solene noted, the build fails when clisp is used as the
cross-compilation host. I've tracked down the breakage to sbcl
revision 83bc16d48d70fdf1b1c0194a0e45753df1fbd480 but haven't made
progress past that.
> Timo
Further comments below.
> Index: Makefile
> ===================================================================
> RCS file: /cvs/ports/lang/sbcl/Makefile,v
> retrieving revision 1.38
> diff -u -p -u -p -r1.38 Makefile
> --- Makefile 24 Jun 2018 18:33:36 -0000 1.38
> +++ Makefile 3 Dec 2018 07:06:41 -0000
> @@ -6,7 +6,7 @@ USE_WXNEEDED = Yes
>
> COMMENT= compiler and runtime system for ANSI Common Lisp
>
> -V = 1.4.8
> +V = 1.4.14
> DISTNAME= sbcl-${V}-source
> PKGNAME= sbcl-${V}
> WRKDIST= ${WRKDIR}/sbcl-${V}
> @@ -21,7 +21,7 @@ PERMIT_PACKAGE_CDROM= Yes
>
> MASTER_SITES= ${MASTER_SITE_SOURCEFORGE:=sbcl/}
>
> -WANTLIB= c m util
> +WANTLIB= c m util z
>
> PSEUDO_FLAVORS= native_bootstrap
>
> @@ -30,10 +30,10 @@ PSEUDO_FLAVORS= native_bootstrap
> FLAVORS= threads
> FLAVOR?=
>
> -EXTRA_PARAMS=
> +EXTRA_PARAMS= --with-sb-core-compression
> .if ${FLAVOR:Mthreads}
> ONLY_FOR_ARCHS = amd64
> -EXTRA_PARAMS+= --with-sb-thread
> +EXTRA_PARAMS+= --with-sb-thread --with-sb-futex
> WANTLIB+= pthread
> .endif
>
> Index: distinfo
> ===================================================================
> RCS file: /cvs/ports/lang/sbcl/distinfo,v
> retrieving revision 1.16
> diff -u -p -u -p -r1.16 distinfo
> --- distinfo 24 Jun 2018 18:33:36 -0000 1.16
> +++ distinfo 3 Dec 2018 07:06:41 -0000
> @@ -1,2 +1,2 @@
> -SHA256 (sbcl-1.4.8-source.tar.bz2) =
> ziS2K4hIJiHCKKT9bIAjUDMLYmPhsbmEYO60qB1/szU=
> -SIZE (sbcl-1.4.8-source.tar.bz2) = 6040563
> +SHA256 (sbcl-1.4.14-source.tar.bz2) =
> TfJu1E1FWAzuy/nhpunkCV3nPHaZxrlFu+jMhxAwHCw=
> +SIZE (sbcl-1.4.14-source.tar.bz2) = 6187821
> Index: patches/patch-src_runtime_Config_generic-openbsd
> ===================================================================
> RCS file:
> /cvs/ports/lang/sbcl/patches/patch-src_runtime_Config_generic-openbsd,v
> retrieving revision 1.3
> diff -u -p -u -p -r1.3 patch-src_runtime_Config_generic-openbsd
> --- patches/patch-src_runtime_Config_generic-openbsd 8 Mar 2018 15:17:39
> -0000 1.3
> +++ patches/patch-src_runtime_Config_generic-openbsd 3 Dec 2018 07:06:41
> -0000
> @@ -12,12 +12,12 @@ Index: src/runtime/Config.generic-openbs
> # provided with absolutely no warranty. See the COPYING and CREDITS
> # files for more information.
>
> --LINKFLAGS += -export-dynamic -Wl,-z,wxneeded
> +-LINKFLAGS += -Wl,-z,wxneeded
> +LINKFLAGS += -Wl,--export-dynamic -Wl,-z,wxneeded
This whole hunk can be dropped, another makefile fragment already adds
-Wl,--export-dynamic
> OS_LIBS += -lutil
>
> ifdef LISP_FEATURE_SB_THREAD
> -@@ -17,9 +17,7 @@ CFLAGS += -pthread
> +@@ -17,9 +17,7 @@ CFLAGS += -pthread -DOS_THREAD_STACK
> OS_LIBS += -pthread
> endif
>
> Index: patches/patch-src_runtime_GNUmakefile
> ===================================================================
> RCS file: /cvs/ports/lang/sbcl/patches/patch-src_runtime_GNUmakefile,v
> retrieving revision 1.8
> diff -u -p -u -p -r1.8 patch-src_runtime_GNUmakefile
> --- patches/patch-src_runtime_GNUmakefile 8 Mar 2018 15:17:39 -0000
> 1.8
> +++ patches/patch-src_runtime_GNUmakefile 3 Dec 2018 07:06:41 -0000
> @@ -2,7 +2,7 @@ $OpenBSD: patch-src_runtime_GNUmakefile,
> Index: src/runtime/GNUmakefile
> --- src/runtime/GNUmakefile.orig
> +++ src/runtime/GNUmakefile
> -@@ -30,7 +30,7 @@ __LDFLAGS__ =
> +@@ -34,7 +34,7 @@ __LDFLAGS__ =
>
> include ../../output/prefix.def
>
> Index: patches/patch-src_runtime_bsd-os_c
> ===================================================================
> RCS file: patches/patch-src_runtime_bsd-os_c
> diff -N patches/patch-src_runtime_bsd-os_c
> --- /dev/null 1 Jan 1970 00:00:00 -0000
> +++ patches/patch-src_runtime_bsd-os_c 3 Dec 2018 07:06:41 -0000
> @@ -0,0 +1,63 @@
> +OB$OpenBSD$
> +
> +Index: src/runtime/bsd-os.c
> +--- src/runtime/bsd-os.c.orig
> ++++ src/runtime/bsd-os.c
> +@@ -693,7 +693,57 @@ os_dlsym(void *handle, const char *symbol)
> + return ret;
> + }
> +
> ++#if defined(LISP_FEATURE_SB_THREAD) && defined(LISP_FEATURE_SB_FUTEX) \
> ++ && !defined(LISP_FEATURE_SB_PTHREAD_FUTEX)
> ++
> ++#include <sys/syscall.h>
> ++#include <unistd.h>
> ++
> ++int __syscall(quad_t, ...);
> ++
> ++#define FUTEX_WAIT 1
> ++#define FUTEX_WAKE 2
> ++#define FUTEX_REQUEUE 3
> ++
> ++#define FUTEX_PRIVATE_FLAG 128
> ++
> ++#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG)
> ++#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG)
> ++#define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG)
Why are you adding all of this and using syscall() instead of just
including <sys/futex.h> and calling futex()?
> ++
> ++int
> ++futex_wait(int *lock_word, int oldval, long sec, unsigned long usec)
> ++{
> ++ struct timespec timeout;
> ++ int t;
> ++
> ++ if (sec < 0) {
> ++ t = __syscall(SYS_futex, lock_word, FUTEX_WAIT, oldval, NULL, NULL);
> ++ }
> ++ else {
> ++ timeout.tv_sec = sec;
> ++ timeout.tv_nsec = usec * 1000;
> ++ t = __syscall(SYS_futex, lock_word, FUTEX_WAIT, oldval, &timeout,
> NULL);
> ++ }
> ++ if (t == 0)
> ++ return 0;
> ++ else if (errno == ETIMEDOUT)
> ++ return 1;
> ++ else if (errno == EINTR)
> ++ return 2;
> ++ else
> ++ /* EWOULDBLOCK and others, need to check the lock */
> ++ return -1;
> ++}
> ++
> ++int
> ++futex_wake(int *lock_word, int n)
> ++{
> ++ return (__syscall(SYS_futex, lock_word, FUTEX_WAKE, n, NULL, NULL));
> ++}
> ++
> + #endif
> ++#endif /* __OpenBSD__ */
> +
> + #if defined(LISP_FEATURE_SB_WTIMER) && !defined(LISP_FEATURE_DARWIN)
> + /*
> Index: patches/patch-src_runtime_run-program_c
> ===================================================================
> RCS file: /cvs/ports/lang/sbcl/patches/patch-src_runtime_run-program_c,v
> retrieving revision 1.1
> diff -u -p -u -p -r1.1 patch-src_runtime_run-program_c
> --- patches/patch-src_runtime_run-program_c 8 Mar 2018 15:17:39 -0000
> 1.1
> +++ patches/patch-src_runtime_run-program_c 3 Dec 2018 07:06:41 -0000
> @@ -6,7 +6,7 @@ copy of environ than libc.
> Index: src/runtime/run-program.c
> --- src/runtime/run-program.c.orig
> +++ src/runtime/run-program.c
> -@@ -200,14 +200,18 @@ int spawn(char *program, char *argv[], int sin, int so
> +@@ -202,14 +202,18 @@ int spawn(char *program, char *argv[], int sin, int so
> if (pwd && chdir(pwd) < 0) {
> failure_code = 3;
> } else {
> Index: patches/patch-src_runtime_thread_c
> ===================================================================
> RCS file: patches/patch-src_runtime_thread_c
> diff -N patches/patch-src_runtime_thread_c
> --- /dev/null 1 Jan 1970 00:00:00 -0000
> +++ patches/patch-src_runtime_thread_c 3 Dec 2018 07:06:41 -0000
> @@ -0,0 +1,22 @@
> +$OpenBSD: patch-src_runtime_thread_c,v 1.1 2018/03/08 15:17:39 sthen Exp $
> +
> +Index: src/runtime/thread.c
> +--- src/runtime/thread.c.orig
> ++++ src/runtime/thread.c
> +@@ -684,9 +684,16 @@ create_thread_struct(lispobj initial_function) {
> + * on the alignment passed from os_validate, since that might
> + * assume the current (e.g. 4k) pagesize, while we calculate with
> + * the biggest (e.g. 64k) pagesize allowed by the ABI. */
> ++#if defined(LISP_FEATURE_OPENBSD) && defined(MAP_STACK)
> ++ void *spaces = mmap(0, THREAD_STRUCT_SIZE, PROT_READ|PROT_WRITE,
> ++ MAP_PRIVATE|MAP_ANON|MAP_STACK, -1, 0);
> ++ if(spaces == MAP_FAILED)
> ++ return NULL;
> ++#else
Please don't reintroduce this patch, it is no longer needed.
> + void *spaces = os_validate(IS_THREAD_STRUCT, NULL, THREAD_STRUCT_SIZE);
> + if(!spaces)
> + return NULL;
> ++#endif
> + /* Aligning up is safe as THREAD_STRUCT_SIZE has
> + * THREAD_ALIGNMENT_BYTES padding. */
> + char *aligned_spaces = PTR_ALIGN_UP(spaces, THREAD_ALIGNMENT_BYTES);
> Index: pkg/PLIST
> ===================================================================
> RCS file: /cvs/ports/lang/sbcl/pkg/PLIST,v
> retrieving revision 1.11
> diff -u -p -u -p -r1.11 PLIST
> --- pkg/PLIST 8 Mar 2018 15:17:39 -0000 1.11
> +++ pkg/PLIST 3 Dec 2018 07:06:41 -0000
> @@ -9,6 +9,8 @@ lib/sbcl/contrib/sb-aclrepl.asd
> lib/sbcl/contrib/sb-aclrepl.fasl
> lib/sbcl/contrib/sb-bsd-sockets.asd
> lib/sbcl/contrib/sb-bsd-sockets.fasl
> +lib/sbcl/contrib/sb-capstone.asd
> +lib/sbcl/contrib/sb-capstone.fasl
> lib/sbcl/contrib/sb-cltl2.asd
> lib/sbcl/contrib/sb-cltl2.fasl
> lib/sbcl/contrib/sb-concurrency.asd
