tlibthread: add pthreadperthread mode and use under ASAN - plan9port - [fork] Plan 9 from user space
HTML git clone git://src.adamsgaard.dk/plan9port
DIR Log
DIR Files
DIR Refs
DIR README
DIR LICENSE
---
DIR commit baef953da253314657be9adea8f371bfbf4ba09e
DIR parent 06687f70ba7a5836c2e872648a85a724a5a1d486
HTML Author: Russ Cox <rsc@swtch.com>
Date: Sun, 17 May 2020 08:24:54 -0400
libthread: add pthreadperthread mode and use under ASAN
ASAN can't deal with the coroutine stacks.
In theory we can call into ASAN runtime to let it know about them,
but ASAN still has problems with fork or exit happening from a
non-system stack. Bypass all possible problems by just having
a full OS thread for each libthread thread. The threads are still
cooperatively scheduled within a proc (in thos mode, a group of OS threads).
Setting the environment variable LIBTHREAD=pthreadperthread
will enable the pthreadperthread mode, as will building with
CC9FLAGS='-fsanitize=address' in $PLAN9/config.
This solution is much more general than ASAN - for example if
you are trying to find all the thread stacks in a reproducible crash
you can use pthreadperthread mode with any debugger that
knows only about OS threads.
Diffstat:
M src/libthread/channel.c | 2 +-
M src/libthread/pthread.c | 33 +++++++++++++++++++++++++++++++
M src/libthread/thread.c | 70 ++++++++++++++++++++++++++++---
M src/libthread/threadimpl.h | 33 +++++++++++++++++++++-----------
4 files changed, 120 insertions(+), 18 deletions(-)
---
DIR diff --git a/src/libthread/channel.c b/src/libthread/channel.c
t@@ -141,7 +141,7 @@ altdequeue(Alt *a)
delarray(ar, i);
return;
}
- fprint(2, "cannot find self in altdq\n");
+ fprint(2, "cannot find self in altdequeue\n");
abort();
}
DIR diff --git a/src/libthread/pthread.c b/src/libthread/pthread.c
t@@ -99,6 +99,23 @@ startprocfn(void *v)
pthread_exit(0);
}
+static void
+startpthreadfn(void *v)
+{
+ void **a;
+ Proc *p;
+ _Thread *t;
+
+ a = (void**)v;
+ p = a[0];
+ t = a[1];
+ free(a);
+ t->osprocid = pthread_self();
+ pthread_detach(t->osprocid);
+ _threadpthreadmain(p, t);
+ pthread_exit(0);
+}
+
void
_procstart(Proc *p, void (*fn)(Proc*))
{
t@@ -116,6 +133,22 @@ _procstart(Proc *p, void (*fn)(Proc*))
}
}
+void
+_threadpthreadstart(Proc *p, _Thread *t)
+{
+ void **a;
+
+ a = malloc(3*sizeof a[0]);
+ if(a == nil)
+ sysfatal("_pthreadstart malloc: %r");
+ a[0] = p;
+ a[1] = t;
+ if(pthread_create(&t->osprocid, nil, (void*(*)(void*))startpthreadfn, (void*)a) < 0){
+ fprint(2, "pthread_create: %r\n");
+ abort();
+ }
+}
+
static pthread_key_t prockey;
Proc*
DIR diff --git a/src/libthread/thread.c b/src/libthread/thread.c
t@@ -7,6 +7,7 @@ static uint threadnsysproc;
static Lock threadnproclock;
static Ref threadidref;
static Proc *threadmainproc;
+static int pthreadperthread;
static void addproc(Proc*);
static void delproc(Proc*);
t@@ -176,12 +177,16 @@ _threadcreate(Proc *p, void (*fn)(void*), void *arg, uint stack)
if(stack < (256<<10))
stack = 256<<10;
- if(p->nthread == 0)
+ if(p->nthread == 0 || pthreadperthread)
stack = 0; // not using it
t = threadalloc(fn, arg, stack);
t->proc = p;
- addthreadinproc(p, t);
+ if(p->nthread == 0)
+ p->thread0 = t;
+ else if(pthreadperthread)
+ _threadpthreadstart(p, t);
p->nthread++;
+ addthreadinproc(p, t);
_threadready(t);
return t;
}
t@@ -209,6 +214,29 @@ proccreate(void (*fn)(void*), void *arg, uint stack)
return id;
}
+// For pthreadperthread mode, procswitch flips
+// between the threads.
+static void
+procswitch(Proc *p, _Thread *from, _Thread *to)
+{
+// fprint(2, "procswitch %p %d %d\n", p, from?from->id:-1, to?to->id:-1);
+ lock(&p->schedlock);
+ from->schedrend.l = &p->schedlock;
+ if(to) {
+ p->schedthread = to;
+ to->schedrend.l = &p->schedlock;
+ _procwakeup(&to->schedrend);
+ }
+ if(p->schedthread != from) {
+ if(from->exiting) {
+ unlock(&p->schedlock);
+ _threadpexit();
+ }
+ _procsleep(&from->schedrend);
+ }
+ unlock(&p->schedlock);
+}
+
void
_threadswitch(void)
{
t@@ -216,9 +244,13 @@ _threadswitch(void)
needstack(0);
p = proc();
+
/*print("threadswtch %p\n", p); */
- if(p->thread->stk == nil)
+
+ if(p->thread == p->thread0)
procscheduler(p);
+ else if(pthreadperthread)
+ procswitch(p, p->thread, p->thread0);
else
contextswitch(&p->thread->context, &p->schedcontext);
}
t@@ -346,6 +378,15 @@ procmain(Proc *p)
threadexits(nil);
}
+void
+_threadpthreadmain(Proc *p, _Thread *t)
+{
+ _threadsetproc(p);
+ procswitch(p, t, nil);
+ t->startfn(t->startarg);
+ threadexits(nil);
+}
+
static void
procscheduler(Proc *p)
{
t@@ -401,9 +442,12 @@ Top:
p->nswitch++;
_threaddebug("run %d (%s)", t->id, t->name);
//print("run %p %p %p %p\n", t, *(uintptr*)(t->context.uc.mc.sp), t->context.uc.mc.di, t->context.uc.mc.si);
- if(t->stk == nil)
+ if(t == p->thread0)
return;
- contextswitch(&p->schedcontext, &t->context);
+ if(pthreadperthread)
+ procswitch(p, p->thread0, t);
+ else
+ contextswitch(&p->schedcontext, &t->context);
/*print("back in scheduler\n"); */
goto Top;
}
t@@ -757,10 +801,24 @@ int
main(int argc, char **argv)
{
Proc *p;
+ char *opts;
argv0 = argv[0];
- if(getenv("NOLIBTHREADDAEMONIZE") == nil)
+ opts = getenv("LIBTHREAD");
+ if(opts == nil)
+ opts = "";
+
+ pthreadperthread = (strstr(opts, "pthreadperthread") != nil);
+#ifdef PLAN9PORT_ASAN
+ // ASAN can't deal with the coroutine stack switches.
+ // In theory it has support for informing it about stack switches,
+ // but even with those calls added it can't deal with things
+ // like fork or exit from a coroutine stack.
+ // Easier to just run in pthread-per-thread mode.
+ pthreadperthread = 1;
+#endif
+ if(strstr(opts, "nodaemon") || getenv("NOLIBTHREADDAEMONIZE") == nil)
_threadsetupdaemonize();
threadargc = argc;
DIR diff --git a/src/libthread/threadimpl.h b/src/libthread/threadimpl.h
t@@ -102,6 +102,17 @@ struct Execjob
Channel *c;
};
+struct _Procrendez
+{
+ Lock *l;
+ int asleep;
+#ifdef PLAN9PORT_USING_PTHREADS
+ pthread_cond_t cond;
+#else
+ int pid;
+#endif
+};
+
struct _Thread
{
_Thread *next;
t@@ -112,6 +123,11 @@ struct _Thread
void (*startfn)(void*);
void *startarg;
uint id;
+#ifdef PLAN9PORT_USING_PTHREADS
+ pthread_t osprocid;
+#else
+ int osprocid;
+#endif
uchar *stk;
uint stksize;
int exiting;
t@@ -120,17 +136,7 @@ struct _Thread
char state[256];
void *udata;
Alt *alt;
-};
-
-struct _Procrendez
-{
- Lock *l;
- int asleep;
-#ifdef PLAN9PORT_USING_PTHREADS
- pthread_cond_t cond;
-#else
- int pid;
-#endif
+ _Procrendez schedrend;
};
extern void _procsleep(_Procrendez*);
t@@ -149,6 +155,7 @@ struct Proc
#endif
Lock lock;
int nswitch;
+ _Thread *thread0;
_Thread *thread;
_Thread *pinthread;
_Threadlist runqueue;
t@@ -157,6 +164,8 @@ struct Proc
uint nthread;
uint sysproc;
_Procrendez runrend;
+ Lock schedlock;
+ _Thread *schedthread;
Context schedcontext;
void *udata;
Jmp sigjmp;
t@@ -188,6 +197,8 @@ extern void _threadpexit(void);
extern void _threaddaemonize(void);
extern void *_threadstkalloc(int);
extern void _threadstkfree(void*, int);
+extern void _threadpthreadmain(Proc*, _Thread*);
+extern void _threadpthreadstart(Proc*, _Thread*);
#define USPALIGN(ucp, align) \
(void*)((((uintptr)(ucp)->uc_stack.ss_sp+(ucp)->uc_stack.ss_size)-(align))&~((align)-1))