Two fixes here:

 - Limit the stack size to 1GB, even if the rlimit is greater than that.
 - Introduce the arg_size variable as upwards-growing stacks were being
   accounted for their entire stack size rather than the amount of stack
   they were currently using.

--- linus-2.5/fs/exec.c	Sat Jan 11 16:42:18 2003
+++ parisc-2.5/fs/exec.c	Sun Jan 12 14:57:54 2003
@@ -337,6 +337,7 @@ int setup_arg_pages(struct linux_binprm 
 	struct vm_area_struct *mpnt;
 	struct mm_struct *mm = current->mm;
 	int i;
+	long arg_size;
 
 #ifdef CONFIG_STACK_GROWSUP
 	/* Move the argument and environment strings to the bottom of the
@@ -369,8 +370,15 @@ int setup_arg_pages(struct linux_binprm 
 
 	/* Adjust bprm->p to point to the end of the strings. */
 	bprm->p = PAGE_SIZE * i - offset;
-	stack_base = STACK_TOP - current->rlim[RLIMIT_STACK].rlim_max;
+
+	/* Limit stack size to 1GB */
+	stack_base = current->rlim[RLIMIT_STACK].rlim_max;
+	if (stack_base > (1 << 30))
+		stack_base = 1 << 30;
+	stack_base = PAGE_ALIGN(STACK_TOP - stack_base);
+
 	mm->arg_start = stack_base;
+	arg_size = i << PAGE_SHIFT;
 
 	/* zero pages that were copied above */
 	while (i < MAX_ARG_PAGES)
@@ -378,6 +386,7 @@ int setup_arg_pages(struct linux_binprm 
 #else
 	stack_base = STACK_TOP - MAX_ARG_PAGES * PAGE_SIZE;
 	mm->arg_start = bprm->p + stack_base;
+	arg_size = STACK_TOP - (PAGE_MASK & (unsigned long) mm->arg_start);
 #endif
 
 	bprm->p += stack_base;
@@ -389,8 +398,8 @@ int setup_arg_pages(struct linux_binprm 
 	if (!mpnt)
 		return -ENOMEM;
 
-	if (!vm_enough_memory((STACK_TOP - (PAGE_MASK & (unsigned long) bprm->p))>>PAGE_SHIFT)) {
+	if (!vm_enough_memory(arg_size >> PAGE_SHIFT)) {
 		kmem_cache_free(vm_area_cachep, mpnt);
 		return -ENOMEM;
 	}
 

.