From mav@purdue.edu Thu Apr 23 17:35:43 1987
Received: from arthur.cs.purdue.edu (purdue.edu.ARPA) by anl-mcs.ARPA (4.12/4.9)
	id AA12475; Thu, 23 Apr 87 17:34:04 cst
Date: Thu, 23 Apr 87 16:10:23 EST
From: "Manolis A Vavalis" <mav@purdue.edu>
Message-Id: <8704232110.AA24826@arthur.cs.purdue.edu>
Received: by arthur.cs.purdue.edu; Thu, 23 Apr 87 16:10:23 EST
To: dongarra@anl-mcs.arpa
Subject:   FLEX - SCHEDULE ...
Cc: dla@purdue.edu
Status: RO


	Dear  Jack,

	     Here is the static version for SCHEDULE for FLEX/32.
	We will contact the people at FLEX to give us some 
	information on MMOS (operating system of flex),
	so that we can finish the dynamic case.
	     Also soon we will start implementing on the NCUBE.

					Manolis

------------------>>>>>>>>>>cut  here<<<<<<<<<<<<<<<---------------------------
# To unbundle, sh this file
echo README 1>&2
cat > README  <<'End of README'

	Comments on the FLEX implementation of the SCHEDULER

       1) Only the static case has been implemented.
       2) Notice that all the shared variables should be declared shared, 
	  which implies you can't pass them as routine arguments. 
	  (see file stuff.cf)
       3) Since the Flex configuration at Purdue has 7 processors the
          max number of processors is set equal to 6. 
	  (see comment in putq.cc)
       4) Due to local memories we run part of the schedule on all
          processors.
       5) The trace data are written in file 'fort.7'


					Dimitris Alexandrakis 
					Manolis Vavalis
End of README
echo Makefile 1>&2
cat > Makefile  <<'End of Makefile'
#
# 	Makefile for SCHEDULER
#
FILES = stuff.o ftsubs.o putq.o
xtest  :	$(FILES)
	cf77 -f -o xtest $(FILES) 

stuff.o: stuff.cf
	cf77 -f -c stuff.cf
ftsubs.o: ftsubs.cf
	cf77 -f -c ftsubs.cf
putq.o: putq.cc
	ccc -c putq.cc
End of Makefile
echo sh.h 1>&2
cat > sh.h  <<'End of sh.h'
      shared integer /qdata/parmq(14,1000),phead,ptail,readyq(1000),
     *                       rhead, rtail,qtail 

      shared integer /qsync/ipad(1),qlock(1000),barrier,hrlock,
     *                       trlock,qtlock,done,flock,jpad(1000)

End of sh.h
echo ftsubs.cf 1>&2
cat > ftsubs.cf  <<'End of ftsubs.cf'
      subroutine chekin(jobtag)
      integer jobtag
c***********************************************************************
c
c     this subroutine records problem identified by jobtag is 
c     done to appropriate nodes.  these nodes are recorded in 
c     parmq(i,jobtag)  where 5 .le. i .le. nchks+4
c     checkin consists of decrementing the value in each of these
c     locations by 1.  each of these is done in a critical section
c     protected by qlock(ichek)
c
c     if the value in parmq(2,ichek) is 0 where ichek is a process
c     dependent upon this one then ichek is placed on the readyq
c     by entering the critical section protected by rtlock.  the
c     pointer rtail to the tail of the readyq is  incremented
c     unless task done is to be recorded.  task done is placed on
c     the ready q and the pointer rtail left in place if nchks .eq. 0
c     is found. 
c
c     see the common block description in libopn for more detail.
c     
c***********************************************************************
#include 'sh.h'
c
c     common block description:
c
c     a complete common block description is given in the routine libopn
c
c****************************************************************************
c
c     check to see if this process has completed.  if not do not checkin
c
      if (parmq(4,jobtag) .ne. 1) return
c
c     the process has completed so chekin proceeds
c
      call CFlock(ICFret, 1, 'FLOCK')
          if (jobtag .ge. 500) then
	      write(7,15) 5,parmq(5,jobtag),jobtag,0
	  else
	      write(7,15) 2,jobtag,0
  	  endif
  15      format (4i8)
      call CFulck(ICFret, 1, 'FLOCK')
      nchks = parmq(3,jobtag)
c     write(6,*) ' from chekin ... jobtag = ',jobtag,'nchks ',nchks
c
c     if this is the final process (indicated by nchks .eq. 0) then
c     record task done.  do not advance the tail so task done sequence
c     is set.  all subsequent gtprb queries will leave rhead .eq. rtail
c     with readyq(rhead) .eq. done.
c
      if (nchks .eq. 0) then 
            call CFlock(ICFret, 1, 'TRLOCK')
               readyq(rtail) = done
            call CFulck(ICFret, 1, 'TRLOCK')
            return
      endif
      do 50 j = 5,nchks+4
         mychek = parmq(j,jobtag)   
c                  
c     get unique access to the checkin node mychek
c     and checkin by decrementing the appropriate counter
c
         mchkgo = 1
         call CFlock(ICFret, 1, 'QLOCK')
            parmq(2,mychek) = parmq(2,mychek) - 1
            mchkgo = parmq(2,mychek)
         call CFulck(ICFret, 1, 'QLOCK')
c
c     place mychek on readyq if parmq(2,mychek) is 0
c        and if mychek is not an alias
c
c     write(6,*) ' from chekin2 ... jobtag = ',jobtag,'nchks ',nchks,
c    *           'mychek ',mychek, ' mchkgo ',mchkgo
         if (mchkgo .eq. 0 .and. parmq(5,mychek) .ne. -3333 ) then
         call CFlock(ICFret, 1, 'TRLOCK')
               readyq(rtail) = mychek
               rtail = rtail + 1
c              write(6,*) ' from chekin in lock ',mychek
            call CFulck(ICFret, 1, 'TRLOCK')
         endif
   50 continue
c     write(6,*) ' from chekin3 ... jobtag = ',jobtag
      return
c
c     last card of chekin
c
      end
c
c
c
      subroutine _dep(jobtag,icango,nchks,mychkn)
      integer jobtag,icango,nchks,mychkn(*)
c*************************************************************************
c
c  warning - this routine may only be used by driver in a static definition
c            of the data dependencies in the task.
c
c            see the examples stuff.f and stuffspwn.f for usage.
c
c     this subroutine puts data dependencies for problem on the queue.  
c     no synchronization is necessary because each index of a column of 
c     parmq is associated with a jobtag specified by the user and 
c     associated with a unique schedulable process.  the arguments of 
c     putq specify a process and are placed in a column of jobq according
c     to the menue specified in the common block description given below.
c
c     jobtag is an integer specifying a unique schedulable process
c
c
c     icango is a positive integer specifying how many processes must check in
c            to this process before it can be placed on the readyq.
c
c     nchks is the number of processes that depend upon the completion of 
c           this process.
c
c     mychkn is an integer array specifying the jobtags of the processes
c            which depend upon completion of this process.
c
c*************************************************************************
c
#include 'sh.h'
c
c     common block description:
c
c     for a complete common block description see the subroutine libopn
c
c
c     place process jobtag on the problem queue
c
      qtail = qtail + 1
      next = jobtag
      parmq(1,next) = 2
      parmq(2,next) = icango
      parmq(3,next) = nchks
      parmq(4,next) = 1
c
c     specify identifiers of processes which depend on this one
c
      do 50 j = 1,nchks
         parmq(j+4,next) = mychkn(j)
   50 continue
c
      call CFlock(ICFret, 1, 'FLOCK')
	  write(7,60) 0,jobtag, icango, nchks,
     &         (parmq(j,next), j = 5,nchks+4)
  60      format (20i8)
      call CFulck(ICFret, 1, 'FLOCK')
      return
c
c     last card of dep
c
      end
      integer function gtprb(jobtag)
c**************************************************************************
c
c     this routine gets unique access to the head of the readyq
c     claims the pointer to the next schedulable process if there 
c     is one and returns with a nonzero value when there is
c     a process to schedule. if task done has been recorded the value
c     zero is returned in gtprb.  if a nonzero value is returned in 
c     gtprb, the integer jobtag will contain the identifier of the process
c     that is to be scheduled.
c
c     output parameters
c
c       jobtag an integer containing the next process to be executed
c
c       gtprb  the return value of this integer function is:
c       
c               0 if task done has been posted
c
c               nonzero if a schedulable process has been claimed.	
c
c
c***************************************************************************
      integer ireqst,jobtag
c
#include 'sh.h'
c
c
c     common block description:
c
c     for a complete common block description see the routine libopn
c
c
      ireqst = jobtag
   10 continue
      mhead = -1
      call CFlock(ICFret, 1, 'HRLOCK')
c
c     gain access to head of readyq.  if task done has not been recorded
c     then increment the head of the readyq. otherwise the head pointer
c     is left fixed so the next active process will receive task done.
c
c        print*,'rhead,rtail,rtail-rhead ',rhead,rtail,rtail-rhead
         mtail = rtail
         if (rhead .lt. rtail) then
            mhead = rhead
c           print*,'rhead,rtail,rtail-rhead ',mhead,mtail,mtail-mhead
            rhead = rhead + 1
         endif
      call CFulck(ICFret, 1, 'HRLOCK')
      if (mhead .gt. 0) then
c
c        there was a work unit on the readyq
c
c	 print *, 'mhead > 0'
         jobtag = readyq(mhead)
         call CFlock(ICFret, 1, 'FLOCK')
             if (jobtag .ge. 500) then
	         write(7,15) 4,parmq(5,jobtag),jobtag
	     else
	         write(7,15) 1,jobtag
  	     endif
  15         format (3i8)
         call CFulck(ICFret, 1, 'FLOCK')
c     print *, 'after 1st csection from gtprb * JOBTAG, mhead, rtail',
c    $			jobtag ,mhead,rtail
c
         if (jobtag .ne. done) then
c           write(6,*) ' ne done jobtag ', jobtag
c
c           record the subroutine call identifier in gtprb and return
c           the process identifier in jobtag.
c
            gtprb = parmq(1,jobtag)
c
	else
c
c	  task  done has been indicated. request a return from subroutine work
c         by returning the value 0 in gtprb.
c
            gtprb = 0
c           write(6,*) ' eq done jobtag rhead rtail ', jobtag,rhead,rtail
c
         endif
      else 
c
c	 print *, 'mhead < 0'
         jobtag = readyq(rhead)
         if (jobtag .eq. done) then
c
c           task done has been posted
c
c           write(6,*) ' task done posted rhead rtail ',rhead,rtail
            gtprb = 0
            return
c
         else
c
c           there was not any work on the readyq
c           return if call was from a wait condition
c
            gtprb = 1
c	    print *, 'not any work on readyq', ireqst
            if (ireqst .eq. -4444) return
            go to 10
c
         endif
      endif
      return
c
c     last card of gtprb
c
      end
      subroutine libop1(nproc)
      integer nproc
c************************************************************************
c
c     this routine sets locks and initializes variables
c     and then spawns nproc generic work routines.
c
c     nproc is a positive integer.  care should be taken to 
c           match nproc to the number of physical processors 
c           available.
c
c************************************************************************
c
#include 'sh.h'
c
c     common block description:
c
c     common/qdata/
c     
c     parmq is a two dimensional integer array.  each column of 
c           this array represents a schedulable process.  a process is
c           identified by its jobtag which corresponds to a unique 
c           column of parmq.  a column of parmq has the following 
c           entries
c
c                   parmq(1,jobtag) = a nonzero integer unless this is the
c                                     last process
c      
c                   parmq(2,jobtag) = icango 
c                                     an integer specifying the number
c                                     of processes that must check in
c                                     before this process may scheduled
c                                     (ie. be placed on the ready queue)
c
c                   parmq(3,jobtag) = nchks
c                                     an integer specifying the number
c                                     of processes that this process 
c                                     must checkin to.  identifiers of
c                                     these processes are recorded below.
c                                     if nchks .eq. 0 then completion of 
c                                     this process signifies completion of
c                                     task.
c
c                   parmq(4,jobtag) = reentry point
c                                     if this integer is greater than 1
c                                     then the process 
c                                     jobtag has spawned process below it
c                                     and will resume execution at a label
c                                     identified with the value in this location
c                                     (see routines spawn,enter,wait,prtspn)
c
c                   parmq(5:5+nchks,jobtag) is reserved for identifiers of the
c                                                  nchks
c                                     processes that must wait for completion
c                                     of this process before they can execute.
c
c
c     phead pointer to head of parmq
c
c     ptail pointer to tail of parmq
c
c     readyq a one dimensional integer array that holds the jobtags of those
c            processes that are ready to execute.
c            if readyq(j) .eq. done has been set then a return from subroutine
c            work is indicated.
c
c     rhead is a pointer to the head of readyq
c
c     rtail is a pointer to the tail of readyq
c
c     common/sync/
c
c     qlock is an integer array of locks.  there is one lock for each 
c           column of parmq.  the purpose of this lock is to ensure
c           unique access to a column of parmq during the checkin operation.
c
c     hrlock is an integer lock.  the purpose of this lock is to ensure 
c            unique access to the pointer rhead to the head of the readyq.
c
c     trlock is an integer lock.  the purpose of this lock is to ensure 
c            unique access to the pointer rtail to the tail of the readyq.
c
c     done   is a unique non positive integer set in libopn to indicate
c            task done.
c
      integer FD
c
      done = 0
c
c     set qlocks off
c
      do 100 j = 1,1000
         readyq(j) = -1
         parmq(4,j) = 1
  100 continue
c
c     set readyq locks off
c
c
c     initialize queue pointers
c
      phead = 1
      ptail = 1
      rhead = 1
      rtail = 1
      qtail = 2
      next = 1 
c
c     set lock on pointer to head of readyq so 
c     no process may start until all process and data dependencies
c     have been specified by the user supplied routine driver.
c
c     call CFlock(ICFret, 1, 'HRLOCK')
c
      return
c
c     last card of libop1
c
      end
      subroutine libop2(nproc)
      integer nproc

      shared integer /lib2/ itag(20),ispace(20),jspace(20)           

      integer procid(20), compno
      external work
c
c     now spawn virtual processors.  these generic work routines will
c     assume the identity of any schedulable process specified by driver.
c
      do 150 j = 1,nproc
         itag(j) = j
  150 continue
c
c     write(6,*) ' in libopn2 , before COEND -->  nprocs = ' ,nproc

      COEND
        do 200 j = 1,nproc
	      
	  compno = j+1
          PROCESS(procid(j),work(itag(j),ispace(j),jspace(j)), compno)

c         call m_fork(_work,itag(j),ispace(j),jspace(j))
c         call work(itag(j),ispace(j),jspace(j))
  200   continue
      END COEND
      return
c
c     last card of libopn
c
      end
      subroutine nxtag(mypar,jobtag)
      integer mypar,jobtag
c*************************************************************************
c
c  warning - this routine may only be used by driver in a static definition
c            of the data dependencies in the task.
c
c     this subroutine puts data dependencies for problem on the queue.  
c     no synchronization
c     is necessary because each index of a column of parmq is associated
c     with a jobtag specified by the user and associated with a unique
c     schedulable process.  the arguments of putq specify a process and are
c     placed in a column of jobq according to the menue specified in the
c     common block description given below.
c
c     jobtag is an integer specifying a unique schedulable process
c
c
c     icango is a positive integer specifying how many processes must check in
c            to this process before it can be placed on the readyq.
c
c     nchks is the number of processes that depend upon the completion of 
c           this process.
c
c     mchkin is an integer array specifying the jobtags of the processes
c            which depend upon completion of this process.
c
c*************************************************************************
c
#include 'sh.h'
c
c
c     common block description:
c
c     for a complete common block description see the subroutine libopn
c
c
c
c     place this process on the next slot in the problem queue
c
      call CFlock(ICFret, 1, 'QTLOCK')
         next = qtail
         qtail = qtail + 1
      call CFulck(ICFret, 1, 'QTLOCK')
      call CFlock(ICFret, 1, 'FLOCK')
	  write(7,15) 3,mypar,next
  15      format (3i8)
      call CFulck(ICFret, 1, 'FLOCK')
      jobtag = next
      parmq(1,next) = 2
      parmq(2,next) = 0
      parmq(3,next) = 1     
      parmq(5,next) = mypar
      parmq(2,mypar) = parmq(2,mypar) + 2
      parmq(6,mypar) = parmq(6,mypar) + 1
c
c        indicate that this is not a reenter 
c
         parmq(4,next) = 1
c
c
c
      return
c
c     last card of nxtag
c
      end
      subroutine start2
c
c     this routine allows parallel processing to start after user supplied
c     driver has completed by unlocking the head of the readyq
c
#include 'sh.h'
c
c
c     for common block description see subroutine libopn.
c
c	print *, ' in start2  -- about to unlock hrlock '
c     call CFulck(ICFret, 1, 'HRLOCK')
c
      return
c
c     last card of start2
c
      end
      subroutine wait(jobtag)
c
      integer jobtag
c*****************************************************************************
c
c     this routine will allow process jobtag to continue after
c     spawned processes have all checked in.  it must be called if 
c     processes have been spawned by an executing process through 
c     the use of the subroutines prtspn and spawn.  
c     the required syntax is 
c
c          call prtspn(mytag)
c          do 100 j = 1,nproc
c                 .
c                 . (set parameters to define spawned process)
c                 .
c             call spawn(mytag,subname,<parms>)
c      100 continue
c          call wait(mytag)
c
c     there is an implied barrier at the call to wait.  the statement
c     following the call to wait is not executed until all spawned 
c     processes have completed execution.
c
c*****************************************************************************
c
#include 'sh.h'
      integer nprocs
c
c        
c     decrement the icango counter so this process can be rescheduled
c     as soon as all spawned processes have checked in.
c
      nprocs = parmq(6,jobtag)
  100 continue
      if (parmq(2,jobtag) .eq. nprocs) then
         return
      else
         jdummy = -4444
         call work2(jdummy,idummy)
      endif
      go to 100
c
c     last card of wait
c
      end
      subroutine place(jobtag)
      integer jobtag
c*************************************************************************
c
c
c      this subroutine places a problem on the readyq 
c
c     jobtag is an integer specifying a unique schedulable process
c
c
c     icango is a positive integer specifying how many processes must check in
c            to this process before it can be placed on the readyq.
c
c     nchks is the number of processes that depend upon the completion of 
c           this process.
c
c     mchkin is an integer array specifying the jobtags of the processes
c            which depend upon completion of this process.
c
c*************************************************************************
c
c
#include 'sh.h'
c
c
c     common block description:
c
c     for a complete common block description see the subroutine libopn
c
c
c
c     place this process on readyq if icango is 0
c     when icango .eq. 0 this process does not depend on any
c     others.
c
      icango = parmq(2,jobtag)
c	print *, 'in place - jobtag,icango ', jobtag,icango
      if (icango .eq. 0 ) then
         call CFlock(ICFret, 1, 'TRLOCK')
            readyq(rtail) = jobtag
c	print *, 'in place rtail,readyq(rtail)',rtail,readyq(rtail)
            rtail = rtail + 1
         call CFulck(ICFret, 1, 'TRLOCK')
      endif
      return
      end
      subroutine prtspn(jobtag)
      integer jobtag,nprocs
c*************************************************************************
c
c
c     this subroutine reserves a slot in the problem queue pointed to by 
c     jobtag.  jobtag will be returned to calling routine through
c     the calling sequence.   jobtag will be an alias for the parent 
c     process that has called this routine. the syntax for the calling
c     program should be 
c
c
c          call prtspn(mytag)
c          do 100 j = 1,nproc
c                 .
c                 . (set parameters to define spawned process)
c                 .
c             call spawn(mytag,subname,<parms>)
c      100 continue
c          call wait(mytag)
c
c
c
c*************************************************************************
c
#include 'sh.h'
c
c
c     common block description:
c
c     for a complete common block description see the subroutine libopn
c
c
c
c     place this process on the next slot in the problem queue
c
      call CFlock(ICFret, 1, 'QTLOCK')
         next = qtail
         qtail = qtail + 1
      call CFulck(ICFret, 1, 'QTLOCK')
      jobtag = next
      parmq(1,next) = 2
      parmq(2,next) = 0
      parmq(3,next) = 1     
      parmq(5,next) = -3333
      parmq(6,next) = 0
c
c     the number -3333 indicates that this is an alias for a parent
c     process.
c
c        indicate that this is not a reenter 
c
         parmq(4,next) = 1
c
c
c
      return
c
c     last card of prtspn
c
      end
End of ftsubs.cf
echo putq.cc 1>&2
cat > putq.cc  <<'End of putq.cc'
#include <stdio.h>

#define MAXPARMS	20

       /* structure to save addresses of subroutines for each processor */
struct lparms {
	int	(*subname)();
};
struct lparms lparms[1000];

struct parms	{
	int	(*subname)();
	long	*parms[MAXPARMS];
		};
struct parms indx[1000]; 

int procid[20];

sched_(nprocs,parms)
int *nprocs;
struct parms parms;
/*  
    this procedure obtains nproc physical processors devoted
    to the the execution of the parallel program indicated through parms
    which is a structure whos first entry is a subroutine name and whow
    remaining entries are parameters appearing in the calling sequence
    of that subroutine.
*/
{
        int libopn_(), (*subname)();
	int j, compno;

	bcopy(&parms, &indx[0], sizeof(struct parms));
	bcopy(&indx[0].subname, &subname, sizeof(subname));
/*      

           the subroutine name and prameter list have been copied and 
           placed in a special slot on the parmq      
          
           then libopn is invoked to initialize pointers grab physical
           processors and begin the computation
*/

/*
    Due to local configuration max number of processors has been set 
    equal to 6
*/
      if (*nprocs >= 6) *nprocs = 6;	

      libop1_(nprocs);     /*  do initialization on data structures  */
      coend { 
	     /*  call the driver for each processor in order to get
		 addresses of the subroutines needed by the scheduler */
	for (j = 1; j <= *nprocs; j++) {
	  compno = j+1;
          process(procid[j], 
	       subname(indx[0].parms[0], indx[0].parms[1],
			       indx[0].parms[2], indx[0].parms[3],
			       indx[0].parms[4], indx[0].parms[5],
			       indx[0].parms[6], indx[0].parms[7],
			       indx[0].parms[8], indx[0].parms[9],
			       indx[0].parms[10], indx[0].parms[11],
			       indx[0].parms[12], indx[0].parms[13],
			       indx[0].parms[14], indx[0].parms[15],
			       indx[0].parms[16], indx[0].parms[17],
			       indx[0].parms[18], indx[0].parms[19]),
		compno);
	}
      }
        libop2_(nprocs);  
	return(0);
}
/*
 *   dep_()  --  call dep() only once to initialize data structures
 */
dep_(jobtag,icango,nchks,mychkn)
int *jobtag,*icango,*nchks,mychkn[];
{
	int ptag;

        if ( (ptag = CCptag()) == procid[1] )  /* only #1 will call dep() */
		_dep_(jobtag,icango,nchks,mychkn);
	return(0);
}
putq_(jobtag,parms)
int *jobtag;
struct parms	parms;
/*  
    this procedure puts the descriptor of a schedulable process <jobtag>
    onto the problem queue.  this process will be scheduled for execution
    when its data dependencies have been satisfied (indicated by icango==0).
    the argument parms is a structure whos first entry is a subroutine name 
    and whos remaining entries are parameters appearing in the calling sequence
    of that subroutine.
*/
{
        register int j;
	int ptag, place_();

        j = *jobtag;
	bcopy(&parms, &indx[j], sizeof(struct parms));
 	   /* copy address of subroutine in local memory of processor */
	bcopy(&indx[j].subname, &lparms[j], sizeof(struct lparms));
/*
        first the parms block is copied into the slot pointed to by 
        by jobtag and then this descriptor is placed on the problem 
        queue
*/ 
	if ( (ptag = CCptag()) == procid[1] ) /* only #1 calls place_() */
		place_(jobtag);
	return(0);
}
spawn_(parent,jobtag,parms)    /*  NOT IMPLEMENTED */
int *parent,*jobtag;
struct parms	parms;
/*  
    this procedure puts the descriptor of a schedulable process <jobtag>
    onto the problem queue.  this process will be scheduled for execution
    when its data dependencies have been satisfied (indicated by icango==0).
    the argument parms is a structure whos first entry is a subroutine name 
    and whos remaining entries are parameters appearing in the calling sequence
    of that subroutine.
    
    the action of this procedure differs from putq in that the user does not
    assign jobtags or data dependencies.  a parent may spawn any number of 
    children but these child processes only report to the parent.
*/
{
        register int j,i;
        int nxtag_(), place_(),clone_();
        nxtag_(parent,jobtag);
        j = *jobtag;
        i = *parent;
	bcopy(&parms, &indx[j], sizeof(struct parms));
/*
        first the parms block is copied into the slot pointed to by 
        by jobtag and then this descriptor is placed on the problem 
        queue
*/ 
        if (indx[j].subname == clone_) indx[j].subname = indx[i].subname;
/*
        here we ask if this is a recursive spawning.  if so the name
        clone has been called instead of subname so we replace the name
        clone by subname.
*/
        place_(jobtag);
	return(0);
}
clone_()
{
/*
        this is a dummy routine to satisfy unresolved external
*/
        return(0);
}
work_(id,jobtag,myjob)
int *id,*jobtag,*myjob;
{
        int start2_(),gtprb_();
        register int j;

        j = *id;
        *myjob = gtprb_(jobtag);
        while (*myjob != 0) 
        {
           j = *jobtag;
	   lparms[j].subname(indx[j].parms[0], indx[j].parms[1],
			       indx[j].parms[2], indx[j].parms[3],
			       indx[j].parms[4], indx[j].parms[5],
			       indx[j].parms[6], indx[j].parms[7],
			       indx[j].parms[8], indx[j].parms[9],
			       indx[j].parms[10], indx[j].parms[11],
			       indx[j].parms[12], indx[j].parms[13],
			       indx[j].parms[14], indx[j].parms[15],
			       indx[j].parms[16], indx[j].parms[17],
			       indx[j].parms[18], indx[j].parms[19]);
             chekin_(jobtag); 
             *myjob = gtprb_(jobtag);
        }
        return(0);
}
work2_(jobtag,myjob)
int *jobtag,*myjob;
{
        int gtprb_();
        register int j;
         *myjob = gtprb_(jobtag);
         if (*myjob == 2) 
         {
           j = *jobtag;
	   indx[j].subname(indx[j].parms[0], indx[j].parms[1],
			       indx[j].parms[2], indx[j].parms[3],
			       indx[j].parms[4], indx[j].parms[5],
			       indx[j].parms[6], indx[j].parms[7],
			       indx[j].parms[8], indx[j].parms[9],
			       indx[j].parms[10], indx[j].parms[11],
			       indx[j].parms[12], indx[j].parms[13],
			       indx[j].parms[14], indx[j].parms[15],
			       indx[j].parms[16], indx[j].parms[17],
			       indx[j].parms[18], indx[j].parms[19]);
             chekin_(jobtag); 
          }
          return(0);
}
/* 
 *	bcopy()  --  block copy from from to to, count bytes 
 */
bcopy(from, to, count)
	char *from, *to;
	int count;
{
	while ((count--) > 0)	/* mjm */
		*to++ = *from++;
}
End of putq.cc
echo stuff.cf 1>&2
cat > stuff.cf  <<'End of stuff.cf'
c
c	parallel main   --   Test for Scheduler
c
c	find the maximum element of an array of 2^k elements
c
	program main
c
c	declare shared variables
c
	shared integer /ints/ n,k

	shared real /reals/ a(1000)
 
	EXTERNAL DRIVER
c	
	print *, 'input k, NPROCS'
	read *, k, NPROCS
	n = 2**k
	do 10 i = 1, n
 10	a(i) = i
c
 	CALL SCHED(nprocs, DRIVER)
c
	print *, 'MAIN  >>>>> max = ', a(1)
	stop
	end
c
c
c
 	subroutine driver
c
c	declare shared variables
c
	shared integer /ints/ n,k

	shared real /reals/ a(1000)
 
c
c	declare local variables
c
	integer mychkn(10), icango, ncheks, jobtag
	EXTERNAL MAXI
c
	job = 0
	do 100 j = 1, k
	    istep = 2**j
	    istop = n+1-istep
	    idstep = n / (2**j)

	    do 200 i = 1, istop, istep
		job = job + 1
		if (mod(job,2) .eq. 0) idstep = idstep-1
		inext = i + 2**(j-1) 
 		JOBTAG = job
		n2 = n/2
		if (job .le. n2) then
 			ICANGO = 0
		else
 			ICANGO = 2
		endif
		if (j .eq. k) then
		    NCHEKS = 0
		else
 		    NCHEKS = 1
		    MYCHKN(1) = job + idstep 
		endif
c 
 		CALL DEP(jobtag,icango,ncheks,mychkn)
 		CALL PUTQ(jobtag, MAXI,a(i),a(inext),a(i))
c
c		call maxi(a(i),a(inext),a(i))
c
  200	    continue
  100	continue
	return
	end
c
c	find maximum of x & y -- return in xymax
c
	subroutine maxi(x,y,xymax)	
	real x, y, xymax
c
	print *, 'in max(a,b,max)  >>  a,b is: ',x,y
	if (x .le. y) then
		xymax = y
	else
		xymax = x
	endif
	print *, ' *** max  = ',xymax
	return
	end
End of stuff.cf
echo trace.16 1>&2
cat > trace.16  <<'End of trace.16'
       0       1       0       1       9
       0       2       0       1       9
       0       3       0       1      10
       0       4       0       1      10
       0       5       0       1      11
       0       6       0       1      11
       0       7       0       1      12
       0       8       0       1      12
       0       9       2       1      13
       0      10       2       1      13
       0      11       2       1      14
       0      12       2       1      14
       0      13       2       1      15
       0      14       2       1      15
       0      15       2       0
       1       1
       1       6
       1       2
       1       3
       1       4
       1       5
       2       1       0
       2       6       0
       2       2       0
       2       3       0
       2       4       0
       2       5       0
       1       7
       1       8
       1       9
       1      10
       1      11
       2       7       0
       2       8       0
       2       9       0
       2      10       0
       2      11       0
       1      12
       1      13
       2      12       0
       1      14
       2      13       0
       2      14       0
       1      15
       2      15       0
End of trace.16
echo trace.32 1>&2
cat > trace.32  <<'End of trace.32'
       0       1       0       1      17
       0       2       0       1      17
       0       3       0       1      18
       0       4       0       1      18
       0       5       0       1      19
       0       6       0       1      19
       0       7       0       1      20
       0       8       0       1      20
       0       9       0       1      21
       0      10       0       1      21
       0      11       0       1      22
       0      12       0       1      22
       0      13       0       1      23
       0      14       0       1      23
       0      15       0       1      24
       0      16       0       1      24
       0      17       2       1      25
       0      18       2       1      25
       0      19       2       1      26
       0      20       2       1      26
       0      21       2       1      27
       0      22       2       1      27
       0      23       2       1      28
       0      24       2       1      28
       0      25       2       1      29
       0      26       2       1      29
       0      27       2       1      30
       0      28       2       1      30
       0      29       2       1      31
       0      30       2       1      31
       0      31       2       0
       1       1
       1       6
       1       2
       1       3
       1       4
       1       5
       2       1       0
       2       6       0
       2       2       0
       2       3       0
       2       4       0
       2       5       0
       1       7
       1       8
       1       9
       1      10
       1      11
       1      12
       2       7       0
       2       8       0
       2       9       0
       2      10       0
       2      11       0
       2      12       0
       1      13
       1      14
       1      15
       1      16
       1      17
       1      18
       2      13       0
       2      14       0
       2      15       0
       2      16       0
       2      17       0
       2      18       0
       1      19
       1      20
       1      21
       1      22
       1      23
       1      24
       2      19       0
       2      20       0
       2      21       0
       2      22       0
       2      23       0
       2      24       0
       1      25
       1      26
       1      27
       1      28
       2      25       0
       2      26       0
       2      27       0
       2      28       0
       1      29
       1      30
       2      29       0
       2      30       0
       1      31
       2      31       0
End of trace.32
echo done ... 1>&2


.