%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: mpi-report.dvi %%CreationDate: Tue Nov 2 11:21:35 1993 %%Pages: 202 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips mpi-report %DVIPSSource: TeX output 1993.11.02:1120 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72 mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1} ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{ statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{ pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get} B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D }B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258437 52099154 1000 300 300 (/ogi/staff/otto/papers/Editor/MPI/draft7/mpi-report.dvi) @start /Fa 2 63 df<0000038000000F0000003C000000F0000003C000000F0000003C 000000F0000003C000000F0000003C000000F0000000F00000003C0000000F00000003C0 000000F00000003C0000000F00000003C0000000F00000003C0000000F00000003801918 7D9520>60 D 62 D E /Fb 2 82 df<03FFC00FFFF030E1F860C07CC0C03C81C01C01C01801C01801C0 30018020018040038380033E000370000700000600000600000E00000C00000C00001800 0016157F9318>80 D<003F0000FFC00307E00401E00800F01000F0300070600070600070 E00070E00060E00060F000C0F000807C31807FE3001F06000008000070003FC0107FC030 0FF86001FFC0003F0014187D931B>I E /Fc 3 82 df<03C00FF01FF83FFC7FFE7FFEFF FFFFFFFFFFFFFF7FFE7FFE3FFC1FF80FF003C010107E9115>15 D<003FFF0001FFFFC007 1E0FE0081C03F0381C01F8703C00F8603C0078C03C0078003C0078003800700038007000 7800E0007800E0007801C0007001800070020000F00C0000E0300000E7C00001EF000001 E0000001C0000001C0000003800000038000000780000007000000070000000E0000000C 0000001D1E809B1D>80 D<0003FC00000FFF0000303F8000400F80018007C0030007C006 0003E00E0003E01C0001E01C0001E0380001E0380001E0700001E0700001E0F00001C0F0 0001C0F00003C0F0000380F0000380F8000700F80006007C030E007F0C1C003FF818000F C020000000400000018000000E00001FF800203FF000E007FE01C000FFC180001FFE0000 03F8001B227C9C22>I E /Fd 1 1 df0 D E /Fe 1 106 df<040C0000000000705898983030606464683006127E910B>105 D E /Ff 5 118 df<03F80FFC1C1C380870006000E000E000E000E00060007000380E1C 1E0FFC03F00F107E8F14>99 D110 D<07C01FF03C78701C701CE00EE00EE00EE00EE00EE00E701C783C3C781FF007C00F107E 8F14>I<030007000700070007007FFCFFFC07000700070007000700070007000700070E 070E070E070C03FC00F00F157F9414>116 DI E /Fg 11 90 df22 D<0038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 38003800380038003800380038FFF8FFF8FFF80D63808118>I<00001800003000006000 00E00000C0000180000380000700000600000E00000C00001C0000380000380000700000 700000E00000E00001E00001C00001C0000380000380000380000780000700000700000F 00000E00000E00001E00001E00001E00001C00001C00003C00003C00003C00003C000038 0000780000780000780000780000780000780000780000780000700000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0000070 00007800007800007800007800007800007800007800007800003800003C00003C00003C 00003C00001C00001C00001E00001E00001E00000E00000E00000F000007000007000007 800003800003800003800001C00001C00001E00000E00000E00000700000700000380000 3800001C00000C00000E000006000007000003800001800000C00000E000006000003000 0018157C768121>32 DI<00001C00003C00 00F80001E00003C0000780000F00000E00001E00003C00003C00003C0000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 F00000F00000F00001E00001E00003C0000380000700000E00001C0000780000E00000E0 00007800001C00000E000007000003800003C00001E00001E00000F00000F00000F00000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 7800007800003C00003C00003C00001E00000E00000F000007800003C00001E00000F800 003C00001C167C7B8121>40 D<0018007800F001E003C007800F001F001E003E003C007C 007C007800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F8 00F800F800F800F800F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C00F800F800F800F001F001E003E003C0078007000E001C 003800F000C000F00038001C000E000700078003C003E001E001F000F000F800F800F800 7C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C0E4D798025>60 D62 D80 D89 D E /Fh 7 116 df<60F0F070101020204040040A7D830A>59 D<003E000C000C000C000C0018001800180018073018F0307060706060C060C060C06080 C080C480C4C1C446C838700F177E9612>100 D<03000380030000000000000000000000 00001C002400460046008C000C0018001800180031003100320032001C0009177F960C> 105 D<00180038001000000000000000000000000001C002200430043008600060006000 6000C000C000C000C001800180018001806300E300C60078000D1D80960E>I<383C1E00 44C6630047028100460301008E0703000C0603000C0603000C060600180C0600180C0620 180C0C20180C0C4030180440301807801B0E7F8D1F>109 D<383C0044C6004702004602 008E06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15 >I<07C00C201870187038001E000FC003E000606060E060C0C0C1803F000C0E7E8D10> 115 D E /Fi 6 94 df<07C018303018701C600C600CE00EE00EE00EE00EE00EE00EE00E E00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF00070007000700 070007000700070007000700070007000700070007000700070007007FF00C157E9412> I<0F8030E040708030C038E0384038003800700070006000C00180030006000C08080810 183FF07FF0FFF00D157E9412>I61 D91 D93 D E /Fj 3 55 df0 D<020002000200C218F2783AE00F800F80 3AE0F278C2180200020002000D0E7E8E12>3 D<0003000300060006000C000C00180018 003000300060006000C000C00180018003000300060006000C000C001800180030003000 60006000C0004000101E7B9600>54 D E /Fk 24 122 df<387C7E7E3E0E1E1C78F06007 0B798416>44 D<70F8F8F870000000000000000070F8F8F8700512789116>58 D<7F1FC0FFBFE07F1FC01C07001C07001C07001C07001C07001C07001C07001FFF001FFF 001FFF001C07001C07001C07001C07001C07001C07001C07001C07001C07007F1FC0FFBF E07F1FC013197F9816>72 D<1FFC003FFE007FFF00780F00F00780E00380E00380E00380 E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380F00780 F00780780F007FFF003FFE001FFC0011197E9816>79 D<07E3001FFF003FFF00781F00F0 0700E00700E00700E00000F000007800003F80001FF00007FC0000FE00000F0000070000 0380000380600380E00380E00700F80F00FFFE00FFFC00C7F00011197E9816>83 D<7FFFE0FFFFE0FFFFE0E0E0E0E0E0E0E0E0E0E0E0E000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00007FC000FFE 0007FC0013197F9816>I<1FE0003FF0007FF800783C00300E00000E00000E0003FE001F FE003E0E00700E00E00E00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<03F80FFC1FFE3C1E780C7000E000E000E000E000E000F000700778073E0E1FFC0FF803 F010127D9116>99 D<003F00007F00003F0000070000070000070000070003C7000FF700 1FFF003C1F00780F00700700E00700E00700E00700E00700E00700E00700700F00700F00 3C1F001FFFE00FE7F007C7E014197F9816>I<03E00FF81FFC3C1E780E7007E007FFFFFF FFFFFFE000E000700778073C0F1FFE0FFC03F010127D9116>I<001F00007F8000FF8001 E78001C30001C00001C0007FFF00FFFF00FFFF0001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C0003FFE007FFF003FFE0011197F9816>I< 7E0000FE00007E00000E00000E00000E00000E00000E3C000EFE000FFF000F87800F0380 0E03800E03800E03800E03800E03800E03800E03800E03800E03800E03807FC7F0FFE7F8 7FC7F01519809816>104 D<018003C003C0018000000000000000007FC07FC07FC001C0 01C001C001C001C001C001C001C001C001C001C001C07FFFFFFF7FFF101A7D9916>I<00 3000780078003000000000000000001FF81FF81FF8003800380038003800380038003800 38003800380038003800380038003800380038003800386070F0F0FFE07FC03F800D237E 9916>I108 DI<7E3C00FEFE007FFF000F87800F03800E03800E03800E03800E03800E03800E 03800E03800E03800E03800E03807FC7F0FFE7F87FC7F01512809116>I<03E0000FF800 1FFC003C1E00780F00700700E00380E00380E00380E00380E00380F00780700700780F00 3C1E001FFC000FF80003E00011127E9116>I<7E3E00FEFF007FFF800F83C00F00E00E00 E00E00700E00700E00700E00700E00700E00700E00E00F01E00F83C00FFF800EFF000E3C 000E00000E00000E00000E00000E00000E00007FC000FFE0007FC000141B809116>I114 D<0FEC3FFC7FFCF03CE01C E01C70007F801FF007F8003C600EE00EF00EF81EFFFCFFF8C7E00F127D9116>I<030000 0700000700000700000700007FFF00FFFF00FFFF00070000070000070000070000070000 07000007000007010007038007038007038007870003FE0001FC0000F80011177F9616> I<7E1F80FE3F807E1F800E03800E03800E03800E03800E03800E03800E03800E03800E03 800E03800E03800E0F800FFFF007FBF803E3F01512809116>I<7F1FC0FF9FE07F1FC01C 07000E07000E0E000E0E00070E00071C00071C00039C00039C0003980001B80001B80000 F00000F00000F00000E00000E00000E00001C00079C0007BC0007F80003F00003C000013 1B7F9116>121 D E /Fl 56 123 df<00E001C0038007000E000E001C001C0038003800 38007000700070007000E000E000E000E000E000E000E000E000E000E000E000E0007000 7000700070003800380038001C001C000E000E000700038001C000E00B2A7E9E10>40 DI<00060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 FFFFFFE0FFFFFFE000060000000600000006000000060000000600000006000000060000 0006000000060000000600000006000000060000000600001B1C7E9720>43 DII<010007003F00FF00C700 070007000700070007000700070007000700070007000700070007000700070007000700 070007000700FFF8FFF80D1C7C9B15>49 D<07C01FF03878701C601EE00EC00F400F4007 00070007000F000E001E001C003C007800F001E001C0038007000E001C0038007000FFFF FFFF101C7E9B15>I<001C0000003E0000003E0000002E0000006700000067000000E780 0000C7800000C3800001C3C0000183C0000181C0000381E0000381E0000700F0000700F0 000600F0000E0078000FFFF8000FFFF8001C003C001C003C0018003C0038001E0038001E 0070001F0070000F0070000F00E0000780191D7F9C1C>65 DI<003FC000FFF003C0F00780300F00001E00003C00003C000078 0000780000780000F00000F00000F00000F00000F00000F00000F00000F00000F0000078 00007800007800003C00003C00001E00000F000807801803C07800FFF0003F80151F7D9D 1B>IIII<003F80 01FFF003C0F80780380F00181E00003C00003C0000780000780000780000F00000F00000 F00000F00000F00000F00000F007F8F007F8F000387800387800387800383C00383C0038 1E00380F003807803803C0F801FFF0003F80151F7D9D1C>III75 DIII<003F000001 FFE00003FFF00007C0F8000F807C001E001E003E001F003C000F00780007807800078078 000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F8 0007C078000780780007807C000F803C000F003E001F001F003E000F807C0007C0F80003 FFF00001FFE000003F00001A1F7E9D1F>II82 D<03F8000FFE001C0F00 380700700300600000E00000E00000E00000E00000F000007800007F00003FE0001FFC00 07FE0001FF00001F800007800003C00003C00001C00001C00001C00001C0C00180E00380 F007007C0E001FFC0007F000121F7E9D17>IIIII<78000E007C001E003C003C001E0038000F0070000F00F0000781 E00003C1C00001C3C00001E7800000F70000007E0000003E0000003C0000003C0000007E 00000077000000E7800001E3800003C1C0000381E0000700F0000F00F8000E0078001C00 3C003C003E0078001F0070000F00F0000F80191D7F9C1C>II91 D93 D<0FC03FF07FF87038401C001C001C00FC0FFC3FFC781CE01CE01CE01CF07C7FFC7FDC3F 1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000E000E000E000700070043C1C1FFC0FF8 07E00E127E9112>I<000E000E000E000E000E000E000E000E000E000E000E0F8E1FEE3F FE7C3E700E700EE00EE00EE00EE00EE00EE00EF00E701E7C3E3FFE1FEE0F8E0F1D7E9C15 >I<07C01FE03FF078787018601CFFFCFFFCFFFCE000E000E000700070043C1C3FFC1FF8 07E00E127E9112>I<00FC01FC03FC07000E000E000E000E000E000E000E00FFE0FFE00E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E1D809C0D >I<03C3C00FFFC01FFFC01C3800381C00381C00381C00381C00381C001C38001FF8001F F0003BC0003800003800001FFC001FFF003FFF80700780E001C0E001C0E001C0F003C07C 0F803FFF001FFE0007F800121B7F9115>III107 DIII<03F0000FFC001FFE003C0F0078078070 0380E001C0E001C0E001C0E001C0E001C0F003C07003807807803C0F001FFE000FFC0003 F00012127F9115>II114 D<1FC03FF07FF0F030E000E000F0007F003FC01FE000F0003800388038F078FF F07FE01FC00D127F9110>I<1C001C001C001C001C001C00FFE0FFE01C001C001C001C00 1C001C001C001C001C001C001C001C001C201FF00FF007C00C187F970F>IIII<7003807807003C0E001C1C000E1C0007380003F0 0001E00001C00001E00003F0000738000E18000E1C001C0E00380700700380F003C01212 809113>II<7FFC7FFC7FFC0078 00F000E001E003C0038007000F001E001C003C007800FFFCFFFCFFFC0E127F9112>I E /Fm 3 104 df<3078F06005047C830D>46 D<01E007100C1018083810701070607F80 E000E000E000E000E000E0086010602030C01F000D127B9113>101 D<00F3018F030F06070E0E0C0E1C0E1C0E381C381C381C381C383830383038187818F00F 700070007000E000E0C0C0E1C0C3007E00101A7D9113>103 D E /Fn 24 121 df<78FCFCFCFC7806067D850D>46 D<78FCFCFCFC7800000000000078FCFC FCFC7806127D910D>58 D61 D<001FE02000FFF8E003F80FE007C003E00F8001E01F0000E03E0000E03E0000607E0000 607C000060FC000000FC000000FC000000FC000000FC000000FC000000FC000000FC0000 007C0000607E0000603E0000603E0000C01F0000C00F80018007C0030003F80E0000FFFC 00001FE0001B1C7D9B22>67 D73 D80 D<003FE00001F07C0003C01E000F800F801F0007 C01F0007C03E0003E07E0003F07C0001F07C0001F0FC0001F8FC0001F8FC0001F8FC0001 F8FC0001F8FC0001F8FC0001F8FC0001F87C0001F07C0001F07E0003F03E0003E03E0703 E01F08C7C00F906F8003D07E0001F87C00003FF8080000380800003C1800003FF800001F F800001FF000000FF0000007E0000003C01D247D9B24>I<0FF8001C1E003E0F803E0780 3E07C01C07C00007C0007FC007E7C01F07C03C07C07C07C0F807C0F807C0F807C0780BC0 3E13F80FE1F815127F9117>97 D<03FC000E0E001C1F003C1F00781F00780E00F80000F8 0000F80000F80000F80000F800007800007801803C01801C03000E0E0003F80011127E91 15>99 D<000FF0000FF00001F00001F00001F00001F00001F00001F00001F00001F00001 F001F9F00F07F01C03F03C01F07801F07801F0F801F0F801F0F801F0F801F0F801F0F801 F07801F07801F03C01F01C03F00F0FFE03F9FE171D7E9C1B>I<01FC000F07001C03803C 01C07801C07801E0F801E0F801E0FFFFE0F80000F80000F800007800007C00603C00601E 00C00F038001FC0013127F9116>I<03F8F00E0F381E0F381C07303C07803C07803C0780 3C07801C07001E0F000E0E001BF8001000001800001800001FFF001FFFC00FFFE01FFFF0 7801F8F00078F00078F000787000707800F01E03C007FF00151B7F9118>103 D<1E003F003F003F003F001E00000000000000000000000000FF00FF001F001F001F001F 001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B1E7F9D0E>105 D108 DII<01FC000F07801C01C03C01E07800F07800F0F800F8 F800F8F800F8F800F8F800F8F800F87800F07800F03C01E01E03C00F078001FC0015127F 9118>II114 D<1FD830786018E018E018F000FF807FE07FF01FF807FC007CC01CC01C E01CE018F830CFC00E127E9113>I<0300030003000300070007000F000F003FFCFFFC1F 001F001F001F001F001F001F001F001F001F0C1F0C1F0C1F0C0F08079803F00E1A7F9913 >III120 D E /Fo 88 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8F870000000 000070F8F8F870051C779B18>33 D<4010E038F078E038E038E038E038E038E038E038E0 38E038E03860300D0E7B9C18>I<030600078F00078F00078F00078F00078F00078F007F FFC0FFFFE0FFFFE07FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFFC0FFFFE0FF FFE07FFFC01E3C001E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18>I<380300 7C07807C0780EE0F80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C007C3C00387C00 00780000780000F80000F00001F00001E00001E00003E00003C00003C00007C000078380 0787C00F87C00F0EE00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07C018038013247E 9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C38001C73F01C73F01C E3F00FE3800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00E0F800E0F800E0 7070E0787070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C7C7E3E0E0E0E1C1C38F8 F0C0070E789B18>I<007000F001E003C007800F001E001C003800380070007000700070 00E000E000E000E000E000E000E000E0007000700070007000380038001C001E000F0007 8003C001F000F000700C24799F18>I<6000F00078003C001E000F000780038001C001C0 00E000E000E000E00070007000700070007000700070007000E000E000E000E001C001C0 038007800F001E003C007800F00060000C247C9F18>I<01C00001C00001C00001C000C1 C180F1C780F9CF807FFF001FFC0007F00007F0001FFC007FFF00F9CF80F1C780C1C18001 C00001C00001C00001C00011147D9718>I<00600000F00000F00000F00000F00000F000 00F00000F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F00000F00000F00000F000 00F00000600013147E9718>I<1C3E7E7F3F1F070E1E7CF860080C788518>I<7FFF00FFFF 80FFFF807FFF0011047D8F18>I<3078FCFC78300606778518>I<00030000078000078000 0F80000F00001F00001E00001E00003E00003C00007C0000780000780000F80000F00001 F00001E00003E00003C00003C00007C0000780000F80000F00000F00001F00001E00003E 00003C00003C00007C0000780000F80000F00000F0000060000011247D9F18>I<01F000 07FC000FFE001F1F001C07003803807803C07001C07001C0E000E0E000E0E000E0E000E0 E000E0E000E0E000E0E000E0E000E0F001E07001C07001C07803C03803801C07001F1F00 0FFE0007FC0001F000131C7E9B18>I<01800380038007800F803F80FF80FB8043800380 0380038003800380038003800380038003800380038003800380038003807FFCFFFE7FFC 0F1C7B9B18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0F000E06000E00000 E00000E00001C00001C00003C0000780000F00001E00003C0000780000F00001E00007C0 000F80001E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8001FFE003FFF0078 07807803C07801C03001C00001C00003C0000380000F0003FF0003FE0003FF0000078000 03C00001C00000E00000E00000E0F000E0F000E0F001C0F003C07C07803FFF001FFE0003 F800131C7E9B18>I<001F00003F0000770000770000E70001E70001C700038700078700 0707000E07001E07003C0700380700780700F00700FFFFF8FFFFF8FFFFF8000700000700 000700000700000700000700007FF000FFF8007FF0151C7F9B18>I<1FFF803FFF803FFF 803800003800003800003800003800003800003800003800003BF8003FFE003FFF003C07 801803C00001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F803FFF001FFC 0003F000131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03C038018038000070 0000700000E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000E0F000E07000E070 00E07000E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18>II<03F8000FFE001FFF003E0F803803807001C07001 C07001C07001C03803803C07801FFF0007FC000FFE001F1F003C07807001C0F001E0E000 E0E000E0E000E0E000E07001C07803C03E0F801FFF000FFE0003F800131C7E9B18>I<03 F0000FFC001FFE003C0F00780780700380E001C0E001C0E001C0E001E0E001E07001E078 03E03C0FE01FFFE00FFEE003F0E00000E00001C00001C00001C0300380780780780F0078 3E003FFC001FF00007C000131C7E9B18>I<3078FCFC783000000000000000003078FCFC 78300614779318>I<183C7E7E3C180000000000000000183C7E7E3E1E0E1C3C78F06007 1A789318>I<000300000780001F80003F00007E0001FC0003F00007E0001FC0003F0000 7E0000FC0000FC00007E00003F00001FC00007E00003F00001FC00007E00003F00001F80 00078000030011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE00000000000000000000000 00FFFFE0FFFFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00007E00003F00001F C00007E00003F00001FC00007E00003F00001F80001F80003F00007E0001FC0003F00007 E0001FC0003F00007E0000FC0000F0000060000011187D9918>I<00700000F80000F800 00D80000D80001DC0001DC0001DC00018C00038E00038E00038E00038E00030600070700 0707000707000707000FFF800FFF800FFF800E03800E03801C01C01C01C07F07F0FF8FF8 7F07F0151C7F9B18>65 DI<00F8E003FEE0 07FFE00F07E01E03E03C01E03800E07000E07000E0700000E00000E00000E00000E00000 E00000E00000E00000E000007000007000E07000E03800E03C00E01E01C00F07C007FF80 03FE0000F800131C7E9B18>I<7FF800FFFE007FFF001C0F801C03C01C03C01C01E01C00 E01C00E01C00F01C00701C00701C00701C00701C00701C00701C00701C00701C00F01C00 E01C00E01C01E01C01C01C03C01C0F807FFF00FFFE007FF800141C7F9B18>III<01F1 C003FDC00FFFC01F0FC01C03C03803C03801C07001C07001C0700000E00000E00000E000 00E00000E00000E00FF0E01FF0E00FF07001C07001C07003C03803C03803C01C07C01F0F C00FFFC003FDC001F1C0141C7E9B18>I<7F07F0FF8FF87F07F01C01C01C01C01C01C01C 01C01C01C01C01C01C01C01C01C01C01C01FFFC01FFFC01FFFC01C01C01C01C01C01C01C 01C01C01C01C01C01C01C01C01C01C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>I< 7FFF00FFFF807FFF0001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C0007FFF00FFFF807FFF00111C7D9B18>I<7F07F0FF87F87F07F01C03C01C07801C07 001C0E001C1E001C3C001C38001C70001CF0001DF0001DF0001FB8001FB8001F1C001E1C 001C0E001C0E001C07001C07001C03801C03801C01C07F03F0FF87F87F03F0151C7F9B18 >75 D<7FE000FFE0007FE0000E00000E00000E00000E00000E00000E00000E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00700E00700E0070 0E00700E00707FFFF0FFFFF07FFFF0141C7F9B18>II<7E07F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C01CC1C01CE1C01C E1C01CE1C01C61C01C71C01C71C01C31C01C39C01C39C01C39C01C19C01C19C01C1DC01C 0DC01C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18>I<0FF8003FFE007FFF00780F00 700700F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380 E00380E00380E00380E00380E00380E00380F00780700700780F007FFF003FFE000FF800 111C7D9B18>II<0FF8003FFE007FFF0078 0F00700700F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E0 0380E00380E00380E00380E00380E1E380E1E380F0E78070F700787F007FFF003FFE000F FC00001C00001E00000E00000F0000070000070011227D9B18>I<7FF800FFFE007FFF00 1C0F801C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE001FFE00 1C0F001C07001C03801C03801C03801C03801C03801C039C1C039C1C039C7F01F8FF81F8 7F00F0161C7F9B18>I<03F3801FFF803FFF807C0F80700780E00380E00380E00380E000 007000007800003F00001FF00007FE0000FF00000F800003C00001C00000E00000E06000 E0E000E0E001E0F001C0F80780FFFF80FFFE00E7F800131C7E9B18>I<7FFFF8FFFFF8FF FFF8E07038E07038E07038E0703800700000700000700000700000700000700000700000 700000700000700000700000700000700000700000700000700000700000700007FF0007 FF0007FF00151C7F9B18>IIII<7F8FE0 7F9FE07F8FE00E07000F0700070E00078E00039C0003DC0001F80001F80000F00000F000 00700000F00000F80001F80001DC00039E00038E00070F000707000E07800E03801E03C0 7F07F0FF8FF87F07F0151C7F9B18>II<3F FFE07FFFE07FFFE07001C07003C0700780700700000F00001E00001C00003C0000780000 700000F00001E00001C00003C0000780000700000F00001E00E01C00E03C00E07800E070 00E0FFFFE0FFFFE0FFFFE0131C7E9B18>II<600000F00000F00000F800 007800007C00003C00003C00003E00001E00001F00000F00000F00000F800007800007C0 0003C00003C00003E00001E00001F00000F00000F800007800007800007C00003C00003E 00001E00001E00001F00000F00000F8000078000078000030011247D9F18>II<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C 38070E789E18>I<1FE0003FF8007FFC00781E00300E0000070000070000FF0007FF001F FF007F0700780700E00700E00700E00700F00F00781F003FFFF01FFBF007E1F014147D93 18>I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFFC0 0FC1E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F0070 0F80E00FC1E00FFFC00EFF80063E00151C809B18>I<01FE0007FF001FFF803E07803803 00700000700000E00000E00000E00000E00000E00000E000007000007001C03801C03E03 C01FFF8007FF0001FC0012147D9318>I<001F80003F80001F8000038000038000038000 038000038003E3800FFB801FFF803C1F80380F80700780700380E00380E00380E00380E0 0380E00380E00380700780700780380F803C1F801FFFF00FFBF803E3F0151C7E9B18>I< 01F00007FC001FFE003E0F00380780700380700380E001C0E001C0FFFFC0FFFFC0FFFFC0 E000007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80007F C000FFE000E1E001C0C001C00001C00001C0007FFFC0FFFFC0FFFFC001C00001C00001C0 0001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF 007FFF007FFF00131C7F9B18>I<01E1F007FFF80FFFF81E1E301C0E0038070038070038 07003807003807001C0E001E1E001FFC001FF80039E0003800001C00001FFE001FFFC03F FFE07801F0700070E00038E00038E00038E000387800F07E03F01FFFC00FFF8001FC0015 1F7F9318>I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF80 0FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E0 0E00E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B18>I<03800007C00007C00007C0 000380000000000000000000000000007FC000FFC0007FC00001C00001C00001C00001C0 0001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000FFFF00FFFF 80FFFF00111D7C9C18>I<0038007C007C007C003800000000000000000FFC1FFC0FFC00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C001C6038F078FFF07FE03F800E277E9C18>II<7FE000FFE0007FE00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E0007FFFC0FFFFE07FFFC0131C7E9B18>I<7CE0E000FFFBF8007FFF F8001F1F1C001E1E1C001E1E1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C 1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C007F1F1F00FFBFBF807F1F1F001914 819318>I<7E3E00FEFF807FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC1714809318>I< 01F0000FFE001FFF003E0F803803807001C07001C0E000E0E000E0E000E0E000E0E000E0 F001E07001C07803C03C07803E0F801FFF000FFE0001F00013147E9318>I<7E3E00FEFF 807FFFC00FC1E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00 700F00700F80E00FC1E00FFFC00EFF800E3E000E00000E00000E00000E00000E00000E00 000E00007FC000FFE0007FC000151E809318>I<01E38007FB801FFF803E1F80380F8070 0780700780E00380E00380E00380E00380E00380E00380700780700780380F803C1F801F FF800FFB8003E380000380000380000380000380000380000380000380003FF8003FF800 3FF8151E7E9318>I<7F87E0FF9FF07FBFF803F87803F03003E00003C00003C000038000 0380000380000380000380000380000380000380000380007FFE00FFFF007FFE0015147F 9318>I<07F7003FFF007FFF00780F00E00700E00700E007007C00007FE0001FFC0003FE 00001F00600780E00380E00380F00380F80F00FFFF00FFFC00E7F00011147D9318>I<01 80000380000380000380000380007FFFC0FFFFC0FFFFC003800003800003800003800003 80000380000380000380000380000380400380E00380E00380E001C1C001FFC000FF8000 3E0013197F9818>I<7E07E0FE0FE07E07E00E00E00E00E00E00E00E00E00E00E00E00E0 0E00E00E00E00E00E00E00E00E00E00E00E00E01E00F03E007FFFC03FFFE01FCFC171480 9318>I<7F8FF0FF8FF87F8FF01E03C00E03800E03800E0380070700070700070700038E 00038E00038E00038E0001DC0001DC0001DC0000F80000F80000700015147F9318>II<7F8FF07F9FF0 7F8FF0070700078E00039E0001DC0001F80000F80000700000F00000F80001DC00039E00 038E000707000F07807F8FF0FF8FF87F8FF015147F9318>I<7F8FF0FF8FF87F8FF00E01 C00E03800E0380070380070700070700038700038600038E0001CE0001CE0000CC0000CC 0000DC0000780000780000780000700000700000700000F00000E00079E0007BC0007F80 003F00001E0000151E7F9318>I<3FFFF07FFFF07FFFF07001E07003C0700780000F0000 1E00003C0000F80001F00003C0000780000F00701E00703C0070780070FFFFF0FFFFF0FF FFF014147F9318>I<0007E0001FE0007FE000780000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E00000E00001E0007FC000FF8000FF80007FC00001E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000007800 007FE0001FE00007E013247E9F18>I<60F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0 F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0600424769F18>I<7C0000FF0000FFC00003C00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000F00000 7FC0003FE0003FE0007FC000F00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00003C000FFC000FF00007C000013247E9F18>I E /Fp 66 124 df<003F83F800FF8FF801FF9FF803E03E00038038000780780007807800 078078000780780007807800078078000780780007807800FFFC7FC0FFFC7FC0FFFC7FC0 078078000780780007807800078078000780780007807800078078000780780007807800 078078000780780007807800078078000780780007807800078078000780780007807800 078078001D2380A21C>11 D<000078007C7800FC7801FC7803C000038000078000078000 078000078000078000078000078000078000FFFC78FFFC78FFFC78078078078078078078 078078078078078078078078078078078078078078078078078078078078078078078078 078078078078078078078078152480A31A>I<0000C018000000C018000000C018000001 803000000180300000018030000001803000000300600000030060000003006000000300 60000003006000000600C000000600C000000600C000000600C000000C018000FFFFFFFF C0FFFFFFFFC0001803000000180300000018030000001803000000300600000030060000 00300600000030060000FFFFFFFFC0FFFFFFFFC000600C000000C018000000C018000000 C018000000C0180000018030000001803000000180300000018030000003006000000300 600000030060000003006000000600C000000600C000000600C00000222D7DA229>35 D<007000E001C003C0078007000F000E001E001C003C003C003800780078007800780070 00F000F000F000F000F000F000F000F000F000F000F000F000F000F00070007800780078 00780038003C003C001C001E000E000F000700078003C001C000E000700C327DA413>40 DI< 787878781830306060E0050A7C830E>44 DII<000080000180000180000300000300000300000600000600000600000C00 000C00000C0000180000180000180000300000300000300000600000600000600000C000 00C00000C000018000018000018000018000030000030000030000060000060000060000 0C00000C00000C0000180000180000180000300000300000300000600000600000600000 C00000C00000C0000011317DA418>I<00C001C007C0FFC0FFC0FBC003C003C003C003C0 03C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C0 03C003C003C0FFFFFFFFFFFF10227CA118>49 D<03F0000FFC001FFE003C1F0030078070 07C06003C0E003E0C001E04001E04001E00001E00001E00001E00003C00003C000078000 0780000F00001E00003C0000780000F00001E00001C0000380000700000E00001C000038 0000700000FFFFE0FFFFE0FFFFE013227EA118>I<01F00007FC001FFF003E0F00380780 7003C02003C02003C00003C00003C00003C0000780000780000F00001E0003FC0003F800 03FE00000F000007800003C00003C00001E00001E00001E00001E00001E08001E0C003C0 E003C07007803C0F801FFF000FFC0003F00013237EA118>I<001F00001F00002F00002F 00006F0000EF0000CF0001CF0001CF00038F00038F00078F00070F000F0F000E0F001E0F 003C0F003C0F00780F00780F00F00F00FFFFF8FFFFF8FFFFF8000F00000F00000F00000F 00000F00000F00000F00000F00000F0015217FA018>I<3FFF803FFF803FFF803C00003C 00003C00003C00003C00003C00003C00003C00003C00003CF8003FFE003FFF003F0F803E 07803C03C03803C00001E00001E00001E00001E00001E00001E00001E04003C04003C0E0 03C07007807C1F003FFE000FFC0003F00013227EA018>I<007E0001FF0003FF0007C100 0F00001E00001E00003C00003C000078000078000078F800F1FE00F7FF00FF0F80FC0780 F803C0F803C0F801C0F001E0F001E0F001E0F001E0F001E07001E07001E07801E07803C0 3803C03C03801C07801F0F000FFE0007FC0001F00013237EA118>II<01F00007FC000FFE001E 0F003C07803C07807803C07803C07803C07803C07803C03803803C07801E0F000F1E0007 FC0003F8000FFE001E0F003C07807803C07803C0F001E0F001E0F001E0F001E0F001E0F0 01E07803C07803C03C07803E0F801FFF0007FC0001F00013237EA118>I<01F00007FC00 0FFE001E0F003C07003807807803807803C0F003C0F001C0F001E0F001E0F001E0F001E0 F001E0F001E07003E07803E07803E03C07E03E1FE01FFDE00FF1E003E3C00003C00003C0 000380000780000700000F00101E00183C003FF8001FF00007C00013237EA118>I<001F 0000001F0000003F8000003F8000003B8000007BC0000073C0000071C00000F1E00000F1 E00000E0E00001E0F00001E0F00001C0F00003C0780003C078000380780007803C000780 3C0007003C000F001E000F001E000FFFFE001FFFFF001FFFFF001C000F003C0007803C00 078038000780780003C0780003C0700003C0F00001E0F00001E0E00001E01B237EA220> 65 DI<000FF000007FFC0000FFFF0001F01F0003C00700078000000F0000001E 0000003E0000003C0000003C000000780000007800000078000000F0000000F0000000F0 000000F0000000F0000000F0000000F0000000F0000000F0000000780000007800000078 0000003C0000003C0000003E0000001E0000000F0000000780008003C0038001F00F8000 FFFF00007FFC00000FF00019257DA31F>IIII<000FF000003FFE 0000FFFF8001F80F8003E00380078000000F0000001E0000001E0000003C0000003C0000 00780000007800000078000000F0000000F0000000F0000000F0000000F0000000F00000 00F000FFC0F000FFC0F000FFC0780003C0780003C0780003C03C0003C03C0003C01E0003 C01E0003C00F0003C0078003C003E003C001F807C000FFFFC0003FFF00000FF8001A257D A321>III76 DII<001FC000007FF00001FF FC0003F07E0007C01F000F800F801F0007C01E0003C03C0001E03C0001E0780000F07800 00F0780000F070000070F0000078F0000078F0000078F0000078F0000078F0000078F000 0078F0000078F0000078780000F0780000F0780000F07C0001F03C0001E03E0003E01E00 03C01F0007C00F800F8007C01F0003F07E0001FFFC00007FF000001FC0001D257DA324> II82 D<00FE0003FFC007FFE00F81E01E00603C00003C00007800007800 007800007800007800007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007 E00001F00000F00000F8000078000078000078000078000078000078C000F0E000F0F801 E07E07C03FFF800FFF0001FC0015257EA31B>IIIII<7800 03E07C0003C03E0007C01E000F800F000F000F801F0007C03E0003C03C0003E07C0001F0 780000F0F0000079F000007DE000003FC000001FC000001F8000000F0000001F8000001F C000003FC000007DE0000079F00000F0F00001F0780001E03C0003C03E0007C01E000780 0F000F000F801F0007801E0003C03C0003E07C0001F0780000F0F80000F81D237FA220> I<07E01FF83FFC381E201E000F000F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF0 0FF83F7FFF3FEF1F8F10167E9517>97 DI<01FC0007FF000FFF801F03803C0180 780000780000700000F00000F00000F00000F00000F00000F00000780000780000780000 3C00401F03C00FFFC007FF8001FC0012167E9516>I<0003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C003E3C00FFBC01FFFC03F0F C03C07C07803C07803C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07803 C07803C03C07C03E0FC01FFFC00FFBC003E3C012237EA219>I<03F00007FC001FFE003E 0F003C0780780380780380F001C0FFFFC0FFFFC0FFFFC0F00000F00000F0000070000078 00007800003C00801F07800FFF8007FF0001F80012167E9516>I<003F00FF01FF03C003 8007800780078007800780078007800780FFF8FFF8FFF807800780078007800780078007 80078007800780078007800780078007800780078007800780102380A20F>I<01F07807 FFF80FFFF81F1F001E0F003C07803C07803C07803C07803C07801E0F001F1F000FFE001F FC0019F0003800003800003C00001FFE001FFFC01FFFE03FFFF07801F07800F8F00078F0 0078F00078F000787800F03E03E01FFFC00FFF8001FC0015217F9518>III<00F000F000F000F000000000000000000000000000000000000000F000F000 F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000 F000F000F000F000F000F000F080F0E3E0FFE07FC01F000C2D83A20D>IIIII<01FC0007FF000FFF801F07C03C01E07800F07800F0700070F00078F00078 F00078F00078F00078F000787800F07800F07C01F03E03E01F07C00FFF8007FF0001FC00 15167F9518>II<03E3C007FBC00FFFC01F0FC03C07C07C03C07803C0F803C0F003C0F003C0F003C0F0 03C0F003C0F003C0F803C07803C07803C03C07C03F0FC01FFFC00FFBC003E3C00003C000 03C00003C00003C00003C00003C00003C00003C00003C00003C012207E9519>II<07F01FFC3FFE3C0E7806780078007C003F003FF01FF80FFC 01FE001F000F000F000FC00FF81EFFFE3FFC0FF010167F9513>I<0F000F000F000F000F 000F00FFF8FFF8FFF80F000F000F000F000F000F000F000F000F000F000F000F000F000F 000F080F1C07FC07F803E00E1C7F9B12>IIII<7801F07C01E03E03C01E07C00F0780 078F0007DE0003FC0001FC0000F80000700000F80001FC0003DC00039E00078F000F0780 1E07801E03C03C01E07800F0F800F81516809516>II<7FFFC07FFFC07FFFC0000F80000F00001E0000 3E00007C0000780000F00001F00003E00003C0000780000F80001F00001E00003C00007C 0000FFFFC0FFFFC0FFFFC012167F9515>II E /Fq 68 125 df<007E1F0001C1B1800303E3C00703C3C00E03C1800E01C0000E01C000 0E01C0000E01C0000E01C0000E01C000FFFFFC000E01C0000E01C0000E01C0000E01C000 0E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C000 0E01C0000E01C0000E01C0007F87FC001A1D809C18>11 D<007E0001C1800301800703C0 0E03C00E01800E00000E00000E00000E00000E0000FFFFC00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 7F87F8151D809C17>I<007FC001C1C00303C00703C00E01C00E01C00E01C00E01C00E01 C00E01C00E01C0FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C07FCFF8151D809C17>I<003F07E0 0001C09C18000380F018000701F03C000E01E03C000E00E018000E00E000000E00E00000 0E00E000000E00E000000E00E00000FFFFFFFC000E00E01C000E00E01C000E00E01C000E 00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00 E01C000E00E01C000E00E01C000E00E01C000E00E01C000E00E01C007FC7FCFF80211D80 9C23>I<6060F0F0F8F86868080808080808101010102020404080800D0C7F9C15>34 D<60F0F8680808081010204080050C7C9C0C>39 D<004000800100020006000C000C0018 001800300030007000600060006000E000E000E000E000E000E000E000E000E000E000E0 00E000600060006000700030003000180018000C000C00060002000100008000400A2A7D 9E10>I<800040002000100018000C000C000600060003000300038001800180018001C0 01C001C001C001C001C001C001C001C001C001C001C00180018001800380030003000600 06000C000C00180010002000400080000A2A7E9E10>I<60F0F070101010102020408004 0C7C830C>44 DI<60F0F06004047C830C>I<0001000300060006 0006000C000C000C0018001800180030003000300060006000C000C000C0018001800180 030003000300060006000C000C000C00180018001800300030003000600060006000C000 C00010297E9E15>I<03C00C301818300C300C700E60066006E007E007E007E007E007E0 07E007E007E007E007E007E007E00760066006700E300C300C18180C3007E0101D7E9B15 >I<030007003F00C7000700070007000700070007000700070007000700070007000700 0700070007000700070007000700070007000F80FFF80D1C7C9B15>I<07C01830201C40 0C400EF00FF80FF807F8077007000F000E000E001C001C00380070006000C00180030006 010C01180110023FFE7FFEFFFE101C7E9B15>I<07E01830201C201C781E780E781E381E 001C001C00180030006007E00030001C001C000E000F000F700FF80FF80FF80FF00E401C 201C183007E0101D7E9B15>I<60F0F0600000000000000000000060F0F06004127C910C> 58 D<60F0F0600000000000000000000060F0F0701010101020204080041A7C910C>I<7F FFFFC0FFFFFFE00000000000000000000000000000000000000000000000000000000000 000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<000600000006000000060000000F000000 0F0000000F00000017800000178000001780000023C0000023C0000023C0000041E00000 41E0000041E0000080F0000080F0000180F8000100780001FFF80003007C0002003C0002 003C0006003E0004001E0004001E000C001F001E001F00FF80FFF01C1D7F9C1F>65 DI<001F808000E061800180198007000780 0E0003801C0003801C00018038000180780000807800008070000080F0000000F0000000 F0000000F0000000F0000000F0000000F0000000F0000000700000807800008078000080 380000801C0001001C0001000E000200070004000180080000E03000001FC000191E7E9C 1E>IIII<001F808000E0618001801980070007800E000380 1C0003801C00018038000180780000807800008070000080F0000000F0000000F0000000 F0000000F0000000F0000000F000FFF0F0000F8070000780780007807800078038000780 1C0007801C0007800E00078007000B800180118000E06080001F80001C1E7E9C21>II I76 DII<00 3F800000E0E0000380380007001C000E000E001C0007003C00078038000380780003C078 0003C0700001C0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0F0 0001E0700001C0780003C0780003C0380003803C0007801C0007000E000E0007001C0003 80380000E0E000003F80001B1E7E9C20>I I82 D<07E0801C1980300580700380600180E00180E0 0080E00080E00080F00000F800007C00007FC0003FF8001FFE0007FF0000FF80000F8000 07C00003C00001C08001C08001C08001C0C00180C00180E00300D00200CC0C0083F80012 1E7E9C17>I<7FFFFFC0700F01C0600F00C0400F0040400F0040C00F0020800F0020800F 0020800F0020000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F 0000001F800003FFFC001B1C7F9B1E>II87 D89 D<08081010202040404040808080808080B0B0F8 F8787830300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC 000F1C00381C00701C00601C00E01C40E01C40E01C40603C40304E801F870012127E9115 >97 DI<07E00C3018783078703060 00E000E000E000E000E000E00060007004300418080C3007C00E127E9112>I<003F0000 070000070000070000070000070000070000070000070000070000070003E7000C170018 0F00300700700700600700E00700E00700E00700E00700E00700E0070060070070070030 0700180F000C370007C7E0131D7E9C17>I<03E00C301818300C700E6006E006FFFEE000 E000E000E00060007002300218040C1803E00F127F9112>I<00F8018C071E061E0E0C0E 000E000E000E000E000E00FFE00E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E007FE00F1D809C0D>I<00038003C4C00C38C01C3880181800381C00 381C00381C00381C001818001C38000C300013C0001000003000001800001FF8001FFF00 1FFF803003806001C0C000C0C000C0C000C06001803003001C0E0007F800121C7F9215> II<18003C003C0018000000000000 000000000000000000FC001C001C001C001C001C001C001C001C001C001C001C001C001C 001C001C001C00FF80091D7F9C0C>I<00C001E001E000C0000000000000000000000000 00000FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180 E001C0E001C0E001C0E001C0E001C0E001C06001807003803003001806000E1C0003F000 12127F9115>II<03C1000C3300180B00300F0070070070 0700E00700E00700E00700E00700E00700E00700600700700700300F00180F000C370007 C700000700000700000700000700000700000700000700003FE0131A7E9116>II<1F9030704030C010C010E010F8007F803FE00FF000F880388018C018C018E010 D0608FC00D127F9110>I<04000400040004000C000C001C003C00FFE01C001C001C001C 001C001C001C001C001C001C101C101C101C101C100C100E2003C00C1A7F9910>IIII<7F8FF00F03800F030007020003840001C8 0001D80000F00000700000780000F800009C00010E00020E000607000403801E07C0FF0F F81512809116>II<7FFC70386038407040F040E041C003 C0038007000F040E041C043C0C380870087038FFF80E127F9112>I II E /Fr 52 124 df<00003FE00000E010000180380003 80780003007800070030000700000007000000070000000E0000000E0000000E000000FF FFE0000E00E0001C01C0001C01C0001C01C0001C01C0001C038000380380003803800038 03800038070000380700007007000070071000700E2000700E2000700E2000E00E2000E0 064000E0038000E0000000C0000001C0000001C000003180000079800000F30000006200 00003C0000001D29829F1A>12 D<000100020004000800100020006000C0018001800300 070006000E000C001C0018003800380030007000700060006000E000E000C000C000C000 C000C000C000C000C000C000C000C000C000C0004000600060002000100010000800102E 79A113>40 D<001000000800000400000600000200000300000300000300000100000180 000180000180000180000180000180000180000380000380000380000300000300000300 000700000700000600000600000E00000C00000C00001C00001800003800003000007000 00600000E00000C0000180000100000300000600000C0000180000300000600000800000 112E80A113>I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I<70F8F8F0E005057B840E>I<000F800030E000E07001C0 700380300380380700380F00780F00780E00781E00781E00703C00F03C00F03C00F03C00 F07801E07801E07801E07801C07003C0F003C0F00380F00780F00700700700700E00701C 003038001870000FC000151F7C9D17>48 D<000200020006000E003C00DC031C001C0038 003800380038007000700070007000E000E000E000E001C001C001C001C0038003800380 03800780FFF80F1E7B9D17>I<001F000061800080E00100E00200700220700420700410 700820F00820F00820F00840E00881E00703C0000380000700000C000018000060000080 000300000400000800401000401000802001807E030047FF0041FE0080FC00807800141F 7C9D17>I<001F0000718000C0C00180C00380E00700E00F00E00F01E01E01E01E01E01E 01E01E01C01C03C01C03C01C03C01C07C01C0F800C0F8006378003C700000F00000E0000 0E00001C00601C00F03800F07000E0600080C0004380003E0000131F7B9D17>57 D<070F1F1F0E0000000000000000000070F8F8F0E008147B930E>I<0000020000000600 0000060000000E0000001E0000001E0000003F0000002F0000004F0000004F0000008F00 00010F0000010F0000020F0000020F0000040F00000C0F0000080F0000100F0000100F00 00200F80003FFF800040078000C007800080078001000780010007800200078002000780 060007801E000F80FF807FF81D207E9F22>65 D<01FFFFC0001E00F0001E0078001E0038 001E003C003C003C003C003C003C003C003C003C0078007800780078007800F0007801E0 00F0078000FFFE0000F00F8000F003C001E001C001E001E001E001E001E001E003C001E0 03C001E003C001E003C001C0078003C00780078007800F0007801E000F007800FFFFE000 1E1F7D9E20>I<0000FE0200078186001C004C0038003C0060003C00C0001C01C0001803 800018070000180F0000181E0000101E0000103C0000003C000000780000007800000078 00000078000000F0000000F0000000F0000000F0000000F0000080700000807000008070 0001003800010038000200180004000C001800060020000381C00000FE00001F217A9F21 >I<01FFFF80001E00E0001E0070001E0038001E001C003C001C003C000E003C000E003C 000E0078000E0078000E0078000E0078000E00F0001E00F0001E00F0001E00F0001E01E0 003C01E0003C01E0003C01E0007803C0007003C0007003C000E003C001C0078001C00780 038007800E0007801C000F007000FFFFC0001F1F7D9E22>I<01FFFFFE001E001C001E00 0C001E0004001E0004003C0004003C0004003C0004003C00040078080800780800007808 000078180000F0300000FFF00000F0300000F0300001E0200001E0200001E0200001E000 1003C0002003C0002003C0004003C00040078000800780018007800100078007000F001F 00FFFFFE001F1F7D9E1F>I<01FFFFFC001E0038001E0018001E0008001E0008003C0008 003C0008003C0008003C00080078001000780800007808000078080000F0100000F03000 00FFF00000F0300001E0200001E0200001E0200001E0200003C0000003C0000003C00000 03C00000078000000780000007800000078000000F800000FFF800001E1F7D9E1E>I<00 00FC040007030C001C00980030007800E0007801C000380380003003800030070000300E 0000301E0000201E0000203C0000003C00000078000000780000007800000078000000F0 000000F000FFF0F0000780F0000780F0000F0070000F0070000F0070000F0070001E0038 001E0018003E001C002E000E00CC000383040000FC00001E217A9F23>I<01FFF3FFE000 1F003E00001E003C00001E003C00001E003C00003C007800003C007800003C007800003C 007800007800F000007800F000007800F000007800F00000F001E00000FFFFE00000F001 E00000F001E00001E003C00001E003C00001E003C00001E003C00003C007800003C00780 0003C007800003C007800007800F000007800F000007800F000007800F00000F801F0000 FFF1FFE000231F7D9E22>I<01FFF0001F00001E00001E00001E00003C00003C00003C00 003C0000780000780000780000780000F00000F00000F00000F00001E00001E00001E000 01E00003C00003C00003C00003C0000780000780000780000780000F8000FFF800141F7D 9E12>I<01FE00007FC0001E0000FC00001E0000F8000017000178000017000178000027 0002F00000270004F00000270004F00000270008F00000470009E00000470011E0000047 0021E00000470021E00000870043C00000838043C00000838083C00000838083C0000103 810780000103820780000103820780000103840780000203840F00000203880F00000203 900F00000203900F00000401E01E00000401E01E00000401C01E00000C01801E00001C01 803E0000FF8103FFC0002A1F7D9E29>77 D<01FFFF80001E00E0001E0070001E0038001E 003C003C003C003C003C003C003C003C003C0078007800780078007800F0007800E000F0 03C000F00F0000FFFC0000F0000001E0000001E0000001E0000001E0000003C0000003C0 000003C0000003C00000078000000780000007800000078000000F800000FFF000001E1F 7D9E1F>80 D<0007E040001C18C0003005800060038000C0038001C00180018001000380 010003800100038001000380000003C0000003C0000003F8000001FF800001FFE000007F F000001FF0000001F8000000780000007800000038000000380020003800200038002000 300060007000600060006000E0007000C000E8038000C606000081F800001A217D9F1A> 83 D<0FFFFFF01E0780E0180780201007802020078020200F0020600F0020400F002040 0F0020801E0040001E0000001E0000001E0000003C0000003C0000003C0000003C000000 78000000780000007800000078000000F0000000F0000000F0000000F0000001E0000001 E0000001E0000001E0000003E00000FFFF00001C1F789E21>I<7FFC1FF807C003C00780 010007800100078001000F0002000F0002000F0002000F0002001E0004001E0004001E00 04001E0004003C0008003C0008003C0008003C0008007800100078001000780010007800 1000F0002000F0002000F0002000F0004000F00040007000800070010000300200003804 00000C18000007E000001D20779E22>II<00F1800389C00707800E03801C03803C038038070078070078070078 0700F00E00F00E00F00E00F00E20F01C40F01C40703C40705C40308C800F070013147C93 17>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E 380F380F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207B9F 15>I<007E0001C1000300800E07801E07801C07003C0200780000780000780000F00000 F00000F00000F00000F0000070010070020030040018380007C00011147C9315>I<0000 780003F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1 C00389C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E 00F00E20F01C40F01C40703C40705C40308C800F070015207C9F17>I<007C01C207010E 011C013C013802780C7BF07C00F000F000F000F0007000700170023804183807C010147C 9315>I<00007800019C00033C00033C000718000700000700000E00000E00000E00000E 00000E0001FFE0001C00001C00001C00001C000038000038000038000038000038000070 0000700000700000700000700000700000E00000E00000E00000E00000C00001C00001C0 000180003180007B0000F300006600003C00001629829F0E>I<003C6000E27001C1E003 80E00700E00F00E00E01C01E01C01E01C01E01C03C03803C03803C03803C03803C07003C 07001C0F001C17000C2E0003CE00000E00000E00001C00001C00301C00783800F0700060 E0003F8000141D7E9315>I<01E0000FE00001C00001C00001C00001C000038000038000 038000038000070000070000071E000763000E81800F01C00E01C00E01C01C03801C0380 1C03801C0380380700380700380700380E10700E20700C20701C20700C40E00CC0600700 14207D9F17>I<00C001E001E001C000000000000000000000000000000E003300230043 804300470087000E000E000E001C001C001C003840388030807080310033001C000B1F7C 9E0E>I<0001800003C00003C00003800000000000000000000000000000000000000000 00003C00004600008700008700010700010700020E00000E00000E00000E00001C00001C 00001C00001C0000380000380000380000380000700000700000700000700000E00000E0 0030E00079C000F180006300003C00001228829E0E>I<01E0000FE00001C00001C00001 C00001C0000380000380000380000380000700000700000703C00704200E08E00E11E00E 21E00E40C01C80001D00001E00001FC00038E00038700038700038384070708070708070 7080703100E03100601E0013207D9F15>I<03C01FC00380038003800380070007000700 07000E000E000E000E001C001C001C001C0038003800380038007000700070007100E200 E200E200E200640038000A207C9F0C>I<1C0F80F0002630C318004740640C004780680E 004700700E004700700E008E00E01C000E00E01C000E00E01C000E00E01C001C01C03800 1C01C038001C01C038001C01C0708038038071003803806100380380E100380380620070 07006600300300380021147C9325>I<1C0F802630C04740604780604700704700708E00 E00E00E00E00E00E00E01C01C01C01C01C01C01C03843803883803083807083803107003 303001C016147C931A>I<007C0001C3000301800E01C01E01C01C01E03C01E07801E078 01E07801E0F003C0F003C0F003C0F00780F00700700F00700E0030180018700007C00013 147C9317>I<01C1E002621804741C04781C04701E04701E08E01E00E01E00E01E00E01E 01C03C01C03C01C03C01C0380380780380700380E003C1C0072380071E00070000070000 0E00000E00000E00000E00001C00001C0000FFC000171D809317>I<00F0400388C00705 800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E00F01C 00F01C00703C00705C0030B8000F380000380000380000700000700000700000700000E0 0000E0000FFE00121D7C9315>I<1C1E002661004783804787804707804703008E00000E 00000E00000E00001C00001C00001C00001C000038000038000038000038000070000030 000011147C9313>I<00FC030206010C030C070C060C000F800FF007F803FC003E000E70 0EF00CF00CE008401020601F8010147D9313>I<018001C0038003800380038007000700 FFF007000E000E000E000E001C001C001C001C0038003800380038207040704070807080 31001E000C1C7C9B0F>I<0E00C03300E02301C04381C04301C04701C08703800E03800E 03800E03801C07001C07001C07001C07101C0E20180E20180E201C1E200C264007C38014 147C9318>I<0E03803307802307C04383C04301C04700C08700800E00800E00800E0080 1C01001C01001C01001C02001C02001C04001C04001C08000E300003C00012147C9315> I<0E00C1C03300E3C02301C3E04381C1E04301C0E04701C060870380400E0380400E0380 400E0380401C0700801C0700801C0700801C0701001C0701001C0602001C0F02000C0F04 000E13080003E1F0001B147C931E>I<0383800CC4401068E01071E02071E02070C040E0 0000E00000E00000E00001C00001C00001C00001C040638080F38080F38100E5810084C6 0078780013147D9315>I<0E00C03300E02301C04381C04301C04701C08703800E03800E 03800E03801C07001C07001C07001C07001C0E00180E00180E001C1E000C3C0007DC0000 1C00001C00003800F03800F07000E06000C0C0004380003E0000131D7C9316>I123 D E /Fs 51 121 df<00030004000800100020006000C00180010003 00060006000C000C001C0018001800380030003000700070006000600060006000E000E0 00E000E000E000E00060006000600060006000300030001000180008000C000400020001 00102E7CA112>40 D<008000C000600020003000180018000C000C000C000E0006000600 0600060006000600060006000600060006000E000E000E000C000C001C00180018003800 3000300060006000C000C001800100030006000C0018003000600080000F2E7FA112>I< 1C3E7E7E3A0202040408081020C0070E7D840D>44 DI<3078F8 787005057C840D>I<007E0001C3000301800701C00E00C00E00E01C00E01C00E03C01E0 3801E07801E07801E07801E07801E07801E0F003C0F003C0F003C0F003C0F003C0F00380 F00780E00780E00700E00700E00E00600E00701C003038003870000FC000131F7C9D17> 48 D<000C001C00FC0F380038003800380038003800700070007000700070007000E000 E000E000E000E000E001C001C001C001C001C001C0038003C0FFFE0F1E7C9D17>I<003F 8000C1E00100F00200780400780400780F007C0F807C0F807C0F00780600780000F80000 F00001E00001C0000380000700000E00001C0000380000600000C0000180000300200600 200800401000403FFFC07FFF80FFFF80161E7E9D17>I<007F000183C00201E00400F007 00F00F00F00F01F00F01F00001E00001E00003C0000380000700000E0000F800000E0000 07000007800007C00003C00007C03007C07807C0F807C0F807C0F00780800F00400E0020 1C0018780007E000141F7D9D17>I<0000600000600000E00001C00003C00005C0000DC0 0009C00011C000238000438000C380008380010380020380040700080700180700100700 200700400700FFFFF0000E00000E00000E00000E00000E00001C00001E0001FFE0141E7E 9D17>I<01803001FFE003FFC003FF0003FC000200000200000200000400000400000400 00047C000587000603800C01800801C00001C00001E00001E00001E00001E07003C0F803 C0F003C0E00380800780400700400E00201C0018700007C000141F7D9D17>I<000F8000 704000C0200180E00301E00701E00E00C01E00001C00003C000038000078F800790E007A 07007C0300F80380F80380F003C0F003C0F003C0F003C0F00780E00780E00780E00700E0 0F00600E00701C0030180018700007C000131F7C9D17>I<2000003FFFE07FFFC07FFF80 400100C00200800200800400000800001000002000004000004000008000018000030000 0300000700000600000E00000E00001E00001C00001C00003C00003C00003C0000780000 780000780000300000131F799D17>I<003F0000C1C00100600200600400300C00300C00 300C00300C00600E00600F80C00FC18007F60003FC0001FC0001FF00063F800C0F801007 C03003C06001C06000C0C000C0C000C0C000C0C00080C0010060030030040018180007E0 00141F7D9D17>I<007E0001C3000301800601C00E01C01C00C03C00E03C00E03C01E078 01E07801E07801E07801E07803E07803E03803C03807C01C0BC00C13C003E38000078000 0780000700000E00600E00F01C00F01800E0300080600041C0003F0000131F7C9D17>I< 07E01838201C401C701CF03CF03C603C0038007000E001C0018003000200060004000400 08000800080008000000000000000000000030007800F800780070000E20799F15>63 D<0000100000001800000038000000380000007800000078000000FC000001BC0000013C 0000033C0000023C0000063C0000043E0000081E0000081E0000101E0000101E0000201E 0000200F0000400F0000400F0000FFFF0000800F0001000F800100078002000780020007 8004000780040007800C0007C03E0007C0FF807FFC1E207E9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800F8007800 F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007C001E003 C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003C003C007 C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E061800380138 007000F801E0007803C0007007800030078000300F0000301F0000301E0000303E000020 3C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000F8000000 F80000007800004078000080780000803C0000803C0001001C0002000E00020006000C00 0300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F0003C007800 78003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00F0001F00 F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01E0007C03 C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FFFFE00020 1F7E9E23>I<07FFFFF8007C0078003C0038003C00180078001800780008007800080078 0008007800080078080800F0100000F0100000F0100000F0300000FFF00000F0700001E0 200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C0002003C0 002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFFF8007C00 78003C0038003C001800780018007800080078000800780008007800080078000800F010 0000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E0200001E020 0001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C00000078000 0007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C00E0003C 01C0003803800018078000180F0000181F0000181E0000183E0000103C0000007C000000 7C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E0780001E0 780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B800380118001E06080 003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C007800007800F000 007800F000007800F000007800F000007800F000007800F00000F001E00000F001E00000 F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003C00001E0 03C00001E003C00001E003C00003C007800003C007800003C007800003C007800003C007 800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C0000 3C00003C0000780000780000780000780000780000780000F00000F00000F00000F00000 F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C00003C00003 C00003C00007800007C000FFFC00131F7F9E10>I<00FFFC0007C00003C00003C0000780 000780000780000780000780000780000F00000F00000F00000F00000F00000F00001E00 001E00001E00001E00001E00001E00003C00303C00783C00F83C00F83800F07800807000 40E00021C0001F000016207D9E17>I<07FFE07FE0007C001F00003C000C00003C001800 00780010000078004000007800800000780100000078020000007804000000F008000000 F010000000F060000000F0F0000000F1F0000000F278000001E478000001E878000001F0 3C000001E03C000001E01E000001E01E000003C00F000003C00F000003C00F000003C007 800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F7E9E23>I<07FF F000007E0000003C0000003C000000780000007800000078000000780000007800000078 000000F0000000F0000000F0000000F0000000F0000000F0000001E0000001E0000001E0 000001E0000001E0008001E0010003C0010003C0010003C0030003C0020003C0060003C0 060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC0007C0000F800003C00 017800003C00017800004E0002F000004E0002F000004E0004F000004E0004F000004E00 08F000004E0008F00000870011E00000870011E00000870021E00000870021E000008700 41E00000838041E00001038083C00001038083C00001038103C00001038203C0000101C2 03C0000101C403C0000201C40780000201C80780000201C80780000201D00780000200F0 0780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A1F7E9E2A>I<07FC 01FFC0003E003E00003E001800003E001800004F001000004F0010000047801000004780 10000043C010000043C010000083C020000081E020000081E020000080F020000080F020 000080782000010078400001007C400001003C400001003C400001001E400001001E4000 02000F800002000F800002000F800002000780000200078000060003800006000300000F 00010000FFE0010000221F7E9E22>I<0003F800001E0E000038070000E0038001C001C0 03C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F83C0000F87C0000F8 7C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0F80003E0780003E0 780003C0780007C07C0007803C000F003C001E001E001C000E0038000700F00003C3C000 00FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F0007800F0007800F800 7800F8007800F8007800F8007800F000F001F000F001E000F003C000F0078000F00F0000 FFF80001E0000001E0000001E0000001E0000001E0000001E0000003C0000003C0000003 C0000003C0000003C0000003C000000780000007C00000FFFC00001D1F7E9E1F>I<0003 F800001E0E000038070000F0038001E001C003C001E0078001E00F0000F00F0000F01F00 00F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C0000F8F80001F0F800 01F0F80001F0F80001F0F80003E0780003E0780003C0780007C0781E07803C210F003C40 9E001E409C000E80F8000740F00003C1C04000FEC0400000E0400000E0800000E1800000 FF800000FF000000FF0000007E0000003C001D297B9F23>I<07FFFC00007C0700003C03 C0003C01E0007801E0007801F0007801F0007801F0007801F0007801E000F003E000F003 C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E01C0001E01C0001E01C 0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C03E0207803E0407C01F 04FFFC0F18000003E01F207E9E21>I<003F040060CC01803C03801C03001C0700180600 080E00080E00080E00080E00000F00000F80000FE00007FE0003FF8001FFC0007FE00007 E00001E00000E00000F00000F04000E04000E04000E04000E06000C0600180E00380F803 00C60C0081F80016217D9F19>I<3FFFFFF03C0780F03007803060078030400F0010400F 0010C00F0010800F0010800F0010800F0010001E0000001E0000001E0000001E0000001E 0000001E0000003C0000003C0000003C0000003C0000003C0000003C0000007800000078 00000078000000780000007800000078000000F0000001F800007FFFE0001C1F7A9E21> IIII<03FFC0FFC0007F007E00003E003800001E003000001E002000000F004000000F0080 00000F81000000078200000007C600000003C400000003E800000001F000000001F00000 0000F000000000F800000000F8000000017C000000023C000000043C0000000C1E000000 081E000000101F000000200F000000400F800000C0078000008007C000010003C0000700 03E0001F8007E000FFE01FFE00221F7F9E22>II<03FFFFC003E007800380078007000F0006001E0004003C000C003C0008 0078000800F0000801F0000001E0000003C0000007800000078000000F0000001E000000 3C0000003C00000078000000F0010001E0010001E0020003C00200078002000F8006000F 0004001E000C003C001C007C003C007800F800FFFFF8001A1F7D9E1C>I<01FC07060E0F 1C0F380E78007000F000F000F000F000E000E000E000E000F0027004300818300FC01014 7C9314>99 D<0000700003F00000F00000700000700000E00000E00000E00000E00000E0 0000E00001C000F9C00305C00E03C01C03C03801C0780380700380F00380F00380F00380 F00380E00700E00700E00700E00700E00700700F00301E00186F000F8FE014207C9F19> I<00F800070E000E07001C0700380380780380700380F00380F00380FFFF80F00000E000 00E00000E00000E00000F001007002003004001C180007E00011147D9314>I<01C003E0 03E003C0018000000000000000000000000003801F800780038003800700070007000700 070007000E000E000E000E000E000E001C001E00FF800B1F7F9E0C>105 D<038F801F90E007A0E003C0600380600780E00700E00700E00700E00700E00700E00E01 C00E01C00E01C00E01C00E01C00E01C01C03801E03C0FFCFF815147E9319>110 D<00FC000387000E01801C00C03800E03800E07000F0F000F0F000F0F000F0F000F0E001 E0E001E0E001C0E003C0F00380700700380E001C1C0007E00014147D9317>I<00800100 01000100030007000F001E00FFF80E000E000E000E001C001C001C001C001C001C003800 38203820382038203840384018800F000D1C7C9B12>116 D<1C0380FC1F803C07801C03 801C0380380700380700380700380700380700380700700E00700E00700E00700E00701E 00701E00703C00305E001F9FC012147B9319>I<1FF0FF03C07801C06001C04000E08000 E180007300007600003C00003C00001C00002E00004E000087000107000203800603800C 01C03E03E0FF07FC18147F9318>120 D E /Ft 63 123 df<00000F80003F8F80007F8F 8000FF8F8001FF8F8003E0000003C0000007C0000007C0000007C0000007C0000007C000 0007C0000007C0000007C0000007C00000FFFF8F80FFFF8F80FFFF8F8007C00F8007C00F 8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F 8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F 8007C00F8007C00F8007C00F8007C00F80192B7FAA20>12 D<001E003C007800F001F001 E003C007C007800F800F001F001F001E003E003E003E007C007C007C007C007C007800F8 00F800F800F800F800F800F800F800F800F800F800F800F800F800F80078007C007C007C 007C007C003E003E003E001E001F001F000F000F80078007C003C001E001F000F0007800 3C001E0F3D7CAC17>40 DI44 DII<007F000001FFC00007FFF0000FFFF8000FC1F8001F007C003F007E003E003E 003C001E007C001F007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F8078000F007C001F007C001F007C001F003E003E003E003E003F007E 001F80FC000FC1F8000FFFF80007FFF00001FFC000007F000019297EA71E>48 D<00180000380000F80007F800FFF800FFF800F8F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F8007FFFF07FFFF07FFFF014287BA71E>I<00FE0003FFC007FFE00FFFF01F03F83C 00FC38007E78003E70003EF0001FF0001F60001F20001F00001F00001F00001F00003E00 003E00007C00007C0000F80001F00001E00003C0000780000F00001E00003C0000780000 F00001E00003C0000780000F00001E00003C00007FFFFF7FFFFF7FFFFF7FFFFF18287EA7 1E>I<007F000001FFC00007FFF0000FFFF8001FC1F8003E007C003C003E0078003E0038 003E0010003E0000003E0000003E0000003C0000007C000000FC000001F8000007F00000 FFE00000FFC00000FFE00000FFF0000001FC0000007C0000003E0000001F0000001F0000 000F8000000F8000000F8000000F8000000F8040000F8060001F00F0001F00F8003F007E 007E003F81FC001FFFF8000FFFF00003FFE000007F000019297EA71E>I<0003F0000007 F0000005F000000DF000000DF000001DF0000039F0000039F0000079F0000079F00000F1 F00000F1F00001E1F00003E1F00003E1F00007C1F00007C1F0000F81F0000F81F0001F01 F0001F01F0003E01F0007C01F0007C01F000F801F000FFFFFF80FFFFFF80FFFFFF80FFFF FF800001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001 F0000001F00019277EA61E>I<3FFFFC3FFFFC3FFFFC3FFFFC3E00003E00003E00003E00 003E00003E00003E00003E00003E00003E00003E3F003EFFC03FFFE03FFFF03FE1F83F80 7C3F003E3E003E00003E00001F00001F00001F00001F00001F00001F00001F20001F6000 3E70003EF8007C7C00FC3F03F81FFFF00FFFE007FF8000FE0018287EA61E>I<000FF000 003FFC0000FFFC0001FFFC0003F80C0007E000000FC000000F8000001F0000001E000000 3E0000003C0000007C0000007C0000007C3FE000F8FFF000F9FFF800FBFFFC00FF807E00 FF003E00FE003F00FC001F00FC001F00FC000F80F8000F80F8000F80F8000F80F8000F80 78000F807C000F807C000F807C000F003E001F003E001F001F003E001F807C000FC1FC00 07FFF80003FFF00001FFC000007F000019297EA71E>II<007F000001FFC00007FFF0000FFFF8001FC1FC003F007E003E003E007E00 3F007C001F007C001F007C001F007C001F007C001F003E003E003E003E001F007C000FC1 F80007FFF00003FFE00003FFE0000FFFF8001FC1FC003F007E003E003E007C001F007C00 1F00F8000F80F8000F80F8000F80F8000F80F8000F80F8000F807C001F007C001F007E00 3F003F007E001FC1FC000FFFF80007FFF00003FFE000007F000019297EA71E>I<007F00 0001FFC00003FFE0000FFFF0000FC1F8001F007C003E007C007C003E007C001E007C001F 00F8001F00F8001F00F8000F00F8000F80F8000F80F8000F80F8000F80F8001F807C001F 807C001F807E003F803E007F803F00FF801FFFEF800FFFCF8007FF8F8003FE1F0000001F 0000001F0000001E0000003E0000003E0000007C0000007C000000F8001801F0001E07E0 003FFFC0001FFF80000FFE000003F8000019297EA71E>I<03FC001FFF807FFFC0FFFFE0 FC07F07001F04000F80000F80000F80000F80000F80001F80001F00007F0000FE0001FC0 001F80003E00003E00007C0000780000F80000F00000F00000F00000F00000F00001F000 01F00001F00000000000000000000000000000000000000000000001F00001F00001F000 01F00001F000152A7DA91C>63 D<0001F000000003F800000003F800000007FC00000007 BC00000007BC0000000F3E0000000F1E0000000F1E0000001F1F0000001E1F0000001E0F 0000003E0F8000003C0F8000003C078000007C07C000007807C00000F803E00000F803E0 0000F003E00001F001F00001F001F00001E001F00003E000F80003E000F80003C000F800 07FFFFFC0007FFFFFC000FFFFFFE000F80003E000F80003E001F00003F001F00001F001E 00001F003E00000F803E00000F803C00000F807C000007C07C000007C078000007C0F800 0003E0F8000003E0232A7EA928>65 DI<0001FF00000FFFE0003FFFF8007FFFF800FE01F801F8003003 F0001007C000000F8000001F8000001F0000003E0000003E0000007E0000007C0000007C 0000007C000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8 000000F8000000F80000007C0000007C0000007C0000007E0000003E0000003E0000001F 0000001F8000000F80000007C0000003F0000401F8001C00FE00FC007FFFFC003FFFF800 0FFFE00001FF001E2C7CAA26>IIII<0001FF00000FFFE0003FFFFC007FFFFE00FF01FE01F8003E03F0000C07C000 000FC000001F8000001F0000003F0000003E0000007E0000007C0000007C0000007C0000 00F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8001FFEF8001F FEF8001FFE7C001FFE7C00003E7C00003E7E00003E3E00003E3F00003E1F00003E1F8000 3E0FC0003E07C0003E03F0003E01F8003E00FF00FE007FFFFE003FFFFC000FFFE00001FF 001F2C7CAA28>III76 DII<0001FC0000000FFF8000003FFFE000007FFFF00001FE03FC0003F800 FE0007E0003F0007C0001F000F80000F801F000007C01F000007C03E000003E03E000003 E07C000001F07C000001F07C000001F078000000F0F8000000F8F8000000F8F8000000F8 F8000000F8F8000000F8F8000000F8F8000000F8F8000000F8F8000000F8F8000000F87C 000001F07C000001F07C000001F07E000003F03E000003E03F000007E01F000007C01F80 000FC00FC0001F8007E0003F0007F0007F0003F800FE0001FE03FC0000FFFFF800003FFF E000000FFF80000001FC0000252C7DAA2C>II82 D<007FC00001FFF80007FFFE000FFFFF001FC07F003F000F007E00 06007C0000007C000000F8000000F8000000F8000000F8000000F8000000FC0000007E00 00007F0000003F8000001FF800000FFF800007FFE00003FFF80000FFFC00000FFE000000 FF0000003F0000001F8000000F8000000FC0000007C0000007C0000007C0000007C00000 07C0000007C000000F8060000F80F0001F00FC003F00FF80FE007FFFFC001FFFF80007FF E00000FF80001A2C7DAA21>IIIII<01FE000FFF803FFFC0 3FFFE03C03F03001F00001F80000F80000F80000F80000F80000F8007FF807FFF81FFFF8 3FE0F87F00F8FC00F8F800F8F800F8F800F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8 151B7E9A1D>97 DI<007FC0 01FFF007FFFC0FFFFC1FC07C1F00083E00007C00007C00007C0000F80000F80000F80000 F80000F80000F80000F800007C00007C00007E00003E00001F000C1FC07C0FFFFC07FFFC 01FFF0007F80161B7E9A1B>I<00003E00003E00003E00003E00003E00003E00003E0000 3E00003E00003E00003E00003E00003E00003E00003E00FC3E03FF3E07FFFE0FFFFE1FC1 FE3F007E3E003E7C003E7C003EFC003EF8003EF8003EF8003EF8003EF8003EF8003EF800 3EFC003E7C003E7C003E3E007E3F00FE1FC1FE0FFFFE07FFBE03FF3E00FC3E172A7EA91F >I<007E0003FF8007FFC00FFFE01F83F03F00F03E00787C00787C003878003CFFFFFCFF FFFCFFFFFCFFFFFCF80000F80000F800007800007C00007C00003E00003F000C1FC07C0F FFFC07FFFC01FFF0007F80161B7E9A1B>I<001FC0007FC000FFC001FFC003F00003E000 07C00007C00007C00007C00007C00007C00007C00007C00007C000FFFE00FFFE00FFFE00 07C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000 07C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000 122A7FA912>I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0003E03E0003E03E0007C 01F0007C01F0007C01F0007C01F0007C01F0007C01F0003E03E0003E03E0001F07C0001F FFC0003FFF80003BFE000038F8000078000000780000003C0000003FFFC0003FFFF8001F FFFC001FFFFE003FFFFF007C007F00F8001F80F8000F80F8000F80F8000F80FC001F807E 003F003F80FE003FFFFE000FFFF80007FFF00000FF80001A287E9A1E>III<007C007C 007C007C007C0000000000000000000000000000000000000000007C007C007C007C007C 007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C 007C007C007C007C007C007C007C007C007C007C40FCF1F8FFF8FFF07FE00F800E3683A9 10>IIIII<007F000001FFC00007FFF0000FFFF8001F C1FC003F007E003E003E007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8 000F80F8000F80F8000F80F8000F807C001F007C001F007E003F003E003E003F007E001F C1FC000FFFF80007FFF00001FFC000007F0000191B7E9A1E>II<00FC3E03FF3E07FFBE0FFFFE1FC1FE3F80FE3E007E7E003E7C003E7C003E F8003EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C003E7E003E3E007E3F00FE 1FC1FE1FFFFE0FFFBE03FF3E01F83E00003E00003E00003E00003E00003E00003E00003E 00003E00003E00003E00003E00003E17277E9A1F>II<03FC001FFF803FFFC07FFFC07C07C0F80080F80000F80000F8 0000FC00007F80007FF8003FFE001FFF0007FF8000FFC0000FE00007E00003E00003E040 03E0E007E0FC0FC0FFFFC07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C000 07C00007C00007C00007C000FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C000 07C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000 07C00007C04007E1C003FFE003FFE001FF8000FC0013227FA116>IIII<7C000FC03E001F803F001F001F803E000F807C0007C0FC 0003E0F80001F1F00001FBE00000FFC000007FC000003F8000001F0000001F0000003F80 00007FC00000FBC00000F3E00001F1F00003E0F80007C07C000F807C000F803E001F001F 003E000F807E000FC0FC0007E01B1B809A1C>III E /Fu 19 121 df<00001E000000003E00 000000FE00000003FE0000003FFE0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00007F FFFFFFC07FFFFFFFC07FFFFFFFC07FFFFFFFC0223879B731>49 D<0007FE0000007FFFE0 0001FFFFF80003FFFFFE0007F01FFF000F8007FF801F0001FFC03E0000FFE07F8000FFF0 7FC0007FF0FFE0007FF8FFF0003FF8FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7F E0003FFC3FC0003FFC1F80003FFC0000003FFC0000003FF80000007FF80000007FF00000 007FF0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00000007 F80000000FF00000001FC00000003F800000007F00000000FC00000001F800000001F000 3C0003E0003C0007C0003C000F000078001E000078003C00007800780000F800F00000F8 01FFFFFFF803FFFFFFF007FFFFFFF00FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FF FFFFFFF0FFFFFFFFE0FFFFFFFFE0FFFFFFFFE026387BB731>I<0003FF8000001FFFF000 007FFFFE0000FE03FF0001F000FF8003C000FFC00780007FE00FF0007FF00FF8007FF01F FC007FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF80FFC007FF007F8 007FF003F0007FF0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007 FC0000001FF000000FFFC000000FFF8000000FFFF800000003FE00000000FF800000007F E00000003FF00000003FF80000003FFC0000001FFC0000001FFE0000001FFE0200001FFF 1FC0001FFF3FE0001FFF7FF0001FFF7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFF F8001FFEFFF0001FFE7FF0003FFC7FE0003FFC3FC0003FF81F80007FF01FE000FFE007FC 03FFC003FFFFFF0001FFFFFE00003FFFF0000007FF800028397CB731>I<00000007C000 0000000FC0000000000FC0000000001FC0000000003FC0000000007FC000000000FFC000 000000FFC000000001FFC000000003FFC000000007FFC00000000FFFC00000000FFFC000 00001EFFC00000003CFFC00000007CFFC0000000F8FFC0000000F0FFC0000001E0FFC000 0003C0FFC0000007C0FFC000000F80FFC000000F00FFC000001E00FFC000003C00FFC000 007C00FFC00000F800FFC00000F000FFC00001E000FFC00003C000FFC00007C000FFC000 0F8000FFC0000F0000FFC0001E0000FFC0003C0000FFC0007C0000FFC000F80000FFC000 FFFFFFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0000001FFC000000001FFC000 000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000 000001FFC000000001FFC0000007FFFFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC0 2A377DB631>I<04000000C00F800007C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE 000FFFFFFC000FFFFFF8000FFFFFF0000FFFFFE0000FFFFF80000FFFFE00000FFFF80000 0F800000000F800000000F800000000F800000000F800000000F800000000F800000000F 800000000F81FF00000F8FFFE0000FBFFFF8000FFE03FE000FF000FF000FC000FF800F80 007FC00F00007FE00700007FF00000003FF00000003FF80000003FF80000003FF8000000 3FFC0000003FFC0600003FFC1F80003FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003F FCFFF0003FFCFFF0003FF8FFE0003FF8FFE0003FF87FC0007FF07F00007FF03C00007FE0 3E0000FFC01F0000FF800FC003FF0007F00FFE0003FFFFFC0001FFFFF000007FFFC00000 0FFC000026397BB731>I<00000FF80000007FFF000003FFFF80000FFC07C0001FE001E0 003FC001F0007F0007F000FF000FF001FE001FF803FC003FF807FC003FF80FFC003FF80F F8003FF81FF8001FF01FF8000FE03FF80007C03FF00000003FF00000007FF00000007FF0 0000007FF00000007FF07FF000FFF0FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE00 1FF0FFFC001FF0FFFC000FF8FFF8000FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000F FEFFF0000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF 7FF0000FFF3FF0000FFF3FF0000FFE3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0F F8001FF807FC001FF003FC001FF001FE003FE000FF80FFC0007FFFFF00001FFFFE000007 FFF8000000FFC00028397CB731>I<1E00000000001F00000000001FF0000000001FFFFF FFFFC01FFFFFFFFFC01FFFFFFFFFC03FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFF FFFE003FFFFFFFFC003FFFFFFFF8003FFFFFFFF0007FFFFFFFF0007C000007E0007C0000 0FC0007800001F80007800001F00007800003E0000F000007E0000F00000FC0000F00001 F80000000003F00000000003E00000000007E0000000000FC0000000000F80000000001F 80000000003F80000000003F00000000007F00000000007F0000000000FF0000000000FE 0000000001FE0000000001FE0000000003FE0000000003FE0000000003FE0000000007FC 0000000007FC0000000007FC000000000FFC000000000FFC000000000FFC000000000FFC 000000000FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC 000000001FFC000000001FFC000000001FFC000000001FFC000000000FF80000000007F0 0000000003E00000002A3B7BB931>I<0001FF8000000FFFF800003FFFFE00007F00FF00 00F8003F8001F0000FC003E0000FE007C00007F007C00007F00FC00003F80FC00003F81F C00003F81FC00003F81FE00003F81FF00003F81FF80003F81FFC0007F01FFF0007F01FFF C007E00FFFF00FE00FFFFC1FC007FFFE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFF FE00007FFFFF00003FFFFF80003FFFFFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFF FC1FC007FFFC3F8001FFFE3F8000FFFE7F00003FFF7F00000FFFFE000003FFFE000001FF FE000000FFFE000000FFFE0000007FFE0000007FFE0000007EFE0000007E7F0000007E7F 000000FC3F800000FC3FC00001F81FE00003F00FF00007E007FE007FC003FFFFFF8000FF FFFE00003FFFF8000003FF800028397CB731>I<0001FF8000001FFFF000007FFFFC0000 FF81FE0001FE007F0007FC003F800FF8003FC00FF8001FE01FF8001FF03FF0001FF03FF0 000FF87FF0000FF87FF0000FFCFFF0000FFCFFF0000FFCFFF0000FFEFFF0000FFEFFF000 0FFEFFF0000FFEFFF0000FFEFFF0000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0001F FF7FF0001FFF3FF0001FFF3FF0001FFF1FF0003FFF0FF8003FFF07F8007FFF03FC006FFF 01FE01CFFF00FFEF8FFF007FFF0FFF000FFE0FFE0000000FFE0000000FFE0000000FFE00 00000FFC0000000FFC03E0001FFC07F0001FF80FF8001FF81FFC001FF01FFC003FF01FFC 003FE01FFC003FC01FF8007FC00FF000FF800FE001FF0007C003FE0007F01FFC0003FFFF F00001FFFFC000007FFF0000000FF8000028397CB731>I<00000001F800000000000000 01F80000000000000003FC0000000000000003FC0000000000000007FE00000000000000 07FE0000000000000007FE000000000000000FFF000000000000000FFF00000000000000 1FFF800000000000001FFF800000000000001FFF800000000000003FFFC0000000000000 3FFFC00000000000007FFFE00000000000007DFFE00000000000007DFFE0000000000000 FDFFF0000000000000F8FFF0000000000001F8FFF8000000000001F07FF8000000000001 F07FF8000000000003F07FFC000000000003E03FFC000000000007E03FFE000000000007 C01FFE00000000000FC01FFF00000000000F801FFF00000000000F800FFF00000000001F 800FFF80000000001F0007FF80000000003F0007FFC0000000003E0003FFC0000000003E 0003FFC0000000007E0003FFE0000000007C0001FFE000000000FC0001FFF000000000F8 0000FFF000000000F80000FFF000000001FFFFFFFFF800000001FFFFFFFFF800000003FF FFFFFFFC00000003FFFFFFFFFC00000003E000003FFC00000007E000003FFE00000007C0 00001FFE0000000FC000001FFF0000000F8000000FFF0000001F8000000FFF8000001F00 00000FFF8000001F00000007FF8000003F00000007FFC000003E00000003FFC000007E00 000003FFE00000FF00000001FFE000FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC 0001FFFFFFF0FFFFFC0001FFFFFFF0443B7DBA4B>65 D<0000001FFF000030000001FFFF E000F000000FFFFFFC01F000007FFFFFFE03F00001FFFE007F87F00003FFE0000FCFF000 0FFF000003FFF0001FFC000001FFF0003FF80000007FF0007FF00000003FF000FFC00000 003FF001FFC00000001FF003FF800000000FF007FF000000000FF00FFF0000000007F00F FE0000000007F01FFE0000000003F01FFE0000000003F03FFC0000000003F03FFC000000 0001F03FFC0000000001F07FFC0000000001F07FF80000000001F07FF80000000000007F F8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000 000000FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FF F8000000000000FFF8000000000000FFF80000000000007FF80000000000007FF8000000 0000007FF80000000000007FFC0000000000F03FFC0000000000F03FFC0000000000F03F FC0000000000F01FFE0000000000F01FFE0000000001E00FFE0000000001E00FFF000000 0001E007FF0000000003C003FF8000000003C001FFC0000000078000FFE00000000F0000 7FF00000001F00003FF80000003E00001FFC0000007C00000FFF000001F8000003FFE000 07F0000001FFFE003FC00000007FFFFFFF000000000FFFFFFC0000000001FFFFF0000000 00001FFF0000003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007 FE00000FF801FF00001FFC00FF80001FFC007FC0001FFC007FE0001FFC003FE0000FF800 3FF0000FF8003FF00007F0003FF00001C0003FF0000000003FF0000000003FF000000000 3FF0000000FFFFF000000FFFFFF000007FF83FF00001FF803FF00007FE003FF0000FF800 3FF0001FF0003FF0003FE0003FF0007FE0003FF0007FE0003FF000FFC0003FF000FFC000 3FF000FFC0003FF000FFC0003FF000FFC0007FF0007FE0007FF0007FE000DFF0003FF003 9FF8001FFC0F0FFFF007FFFE0FFFF001FFFC07FFF0003FE000FFF02C267DA530>97 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007FC001FC00FF8000FE0 0FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE00003FC7FE00003FC7F E00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE0000000FFE0000000FFE0 000000FFE00000007FE00000007FE00000007FE00000003FE00000003FF000003C1FF000 003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E000FFC03FC0003FFFFF 000007FFFC000000FFE00026267DA52D>101 D<00FF00000000FFFF00000000FFFF0000 0000FFFF00000000FFFF0000000007FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF007FC00003FF01FFF80003FF07FFFC0003FF0F03FE0003FF1C01 FF0003FF3001FF8003FF6000FF8003FFE000FFC003FFC000FFC003FF8000FFC003FF8000 FFC003FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3F FFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF303C7CBB37>104 D<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FFFE1C01FF0007FE3001FF 8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003FF8000FFC003FF8000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFF FFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<00FF01FF8000FFFF0FFFF000FFFF 3FFFFC00FFFFFE03FF00FFFFF000FF8003FFC0007FC003FF80003FE003FF00003FF003FF 00001FF803FF00001FFC03FF00000FFC03FF00000FFE03FF00000FFE03FF000007FE03FF 000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF 000007FF03FF000007FF03FF000007FF03FF000007FE03FF000007FE03FF00000FFE03FF 00000FFC03FF00000FFC03FF00001FF803FF00001FF803FF00003FF003FF80003FE003FF C0007FC003FFF001FF8003FFFC07FF0003FF3FFFFC0003FF0FFFF00003FF01FF000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 00000000FFFFFC000000FFFFFC000000FFFFFC000000FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80FFFE707FC007FE60FFE003FEE0FFE0 03FEC0FFE003FFC0FFE003FF807FC003FF807FC003FF803F8003FF800E0003FF00000003 FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF000000FFFFFE0000FFFFFE00 00FFFFFE0000FFFFFE000023267DA529>114 D<00078000000780000007800000078000 00078000000F8000000F8000000F8000000F8000001F8000001F8000003F8000003F8000 007F800000FF800001FF800007FF80001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C00FF8078 00FFC078007FC070003FE0E0001FFFC00007FF800001FF001E377EB626>116 D120 D E /Fv 78 123 df<007F07F001FF1FF003FF3FF007807800070070000F00F0000F00F0000F00F0000F00 F0000F00F0000F00F0000F00F000FFF8FF80FFF8FF80FFF8FF800F00F0000F00F0000F00 F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00 F0000F00F0000F00F0000F00F0000F00F0000F00F0001C20809F1B>11 D<0000F000F8F001F8F003F8F00780000700000F00000F00000F00000F00000F00000F00 000F0000FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00 F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0142180A018>I<00 F8F001F8F003F8F00780F00700F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0FF F8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F 00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01420809F18>I<783C783C783C 783C180C3018301860306030E0700E0A7F9F17>34 D<007000E001C00380078007000E00 1E001E003C003C003C0078007800780078007000F000F000F000F000F000F000F000F000 F000F000F000F000700078007800780078003C003C003C001E001E000E00070007800380 01C000E000700C2E7EA112>40 DI<018001C001800180C183E187F99F7DBE1FF807E007E01FF87DBEF99FE187C1 830180018001C0018010147DA117>I<0006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 00060000FFFFFFF0FFFFFFF0000600000006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 1C207D9A23>I<787878781830306060E0050A7D830D>II< F0F0F0F004047C830D>I<000100030003000600060006000C000C000C00180018001800 300030003000600060006000C000C000C00180018001800300030003000600060006000C 000C000C00180018001800300030003000600060006000C000C000C000102D7DA117>I< 03F00007F8001FFE001E1E003C0F00380700780780780780700380F003C0F003C0F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0780780 7807807807803807003C0F001E1E001FFE000FFC0003F00012207E9E17>I<00C001C00F C0FFC0FFC0F3C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C0FFFEFFFEFFFE0F1F7C9E17>I<07F0000FFC001FFE 00383F00700F00600780E00780E003C04003C04003C00003C00003C00003C00007800007 80000F00000E00001C00003C0000780000E00001C0000380000700000E00001C00003800 00700000FFFFC0FFFFC0FFFFC0121F7E9E17>I<03F0000FFC001FFE003C1F00780F0030 0780200780000780000780000780000F00000F00003E0003FC0003F80003FC00001E0000 0F000007800003800003C00003C00003C00003C08003C0C003C0C00780700F807C1F003F FE000FFC0003F00012207E9E17>I<003E00003E00005E00005E0000DE0001DE00019E00 039E00039E00079E00071E000F1E000E1E001E1E003C1E003C1E00781E00781E00F01E00 FFFFF0FFFFF0FFFFF0001E00001E00001E00001E00001E00001E00001E00001E00141E7F 9D17>I<7FFF007FFF007FFF0078000078000078000078000078000078000078000079F0 007FFC007FFE007F1F007C07007C07807807800003C00003C00003C00003C00003C00003 C00003C0400780600780F00F007C1E003FFC001FF80007E000121F7E9D17>I<007C0001 FE0003FE000782000F00001E00003C00003C000078000078000078F800F3FC00F7FE00FE 1F00FC0F80F80780F80780F003C0F003C0F003C0F003C0F003C07003C07803C07803C078 07803807803C0F001E1E000FFE0007F80001F00012207E9E17>II<03F0000FFC001FFE003E1F003C0F007807807807 807807807807807807807807803C0F001E1E000FFC0007F8000FFC001F3E003C0F007807 80780780F003C0F003C0F003C0F003C0F003C0F003C07807807C0F803E1F001FFE000FFC 0003F00012207E9E17>I<03F00007F8000FFC001E1E003C0F00780700780780F00780F0 0380F003C0F003C0F003C0F003C0F003C0F003C07807C07807C07C0FC03E1FC01FFBC00F F3C007C780000780000780000700000F00001E00201E00307C007FF8003FF0000FC00012 207E9E17>II61 D<001F0000001F0000003F8000003B8000003B8000 007BC0000073C0000071C00000F1E00000E1E00000E0E00001E0F00001E0F00001C0F000 03C0780003C078000380780007803C0007803C0007003C000FFFFE000FFFFE000FFFFE00 1E000F001E000F003C000F803C0007803C000780780007C0780003C0780003C0F00003E0 1B207F9F1E>65 DI<001FC000FFF801FFFC03E03C07800C0F00001E00003E00003C00007C0000780000 780000780000F00000F00000F00000F00000F00000F00000F00000F00000780000780000 7800007C00003C00003E00001E00000F000207800E03E03E01FFFC00FFF0001FC017227D A01D>IIII<001FE000FFF801FFFE03E0 3E07800E0F00001E00003E00003C00007C0000780000780000780000F00000F00000F000 00F00000F00000F00000F003FEF003FE7803FE78001E78001E7C001E3C001E3E001E1E00 1E0F001E07801E03E03E01FFFE00FFF8001FC017227DA01E>III75 DII I<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F 00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003 C0F00003C0F00003C0F00003C0F80007C07800078078000780780007803C000F003C000F 001E001E001F003E000F807C0007E1F80003FFF00000FFC000003F00001A227DA021>I< FFF800FFFF00FFFF80F00FC0F003E0F001E0F000F0F000F0F000F0F000F0F000F0F000F0 F000F0F001E0F003E0F00FC0FFFF80FFFF00FFF800F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000F00000F00000F0000014207B9F1D>I<003F000000 FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F007800078078 00078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F0 0003C0F00003C0F00003C07800078078000780781E07803C0F0F003C0F8F001E079E001F 03FE000F83FC0007E1F80003FFF00000FFF800003F780000007C0000003E0000001E0000 001F0000000F801A277DA021>II<01FC0007FF800FFFC01F03C03C00C03C0000780000780000780000 7800007800007C00003C00003F00001FE0000FFC0007FE0001FF00003F800007C00003C0 0003E00001E00001E00001E00001E00001E00001C0C003C0F007C0FC0F807FFF001FFE00 03F80013227EA019>I IIII<780007807C000F003E001F001E001E000F003C000F807C 000780780003C0F00003E1F00001F1E00000F3C000007FC000007F8000003F0000001F00 00001E0000003F0000007F8000007FC00000F3C00001F1E00001E0F00003C0F80007C078 0007803C000F003E001F001E001E000F003C000F807C0007C0780003C0F00003E01B207F 9F1E>IIII<381C3018301860306030C060F078F078F078F0780E0A799F17>II<07E03FF87FFC701E401F000F000F000F003F07FF1FFF7E0F F80FF00FF00FF00FF83F7FFF3FEF1F8F10147E9316>97 DI<03F00FFC1FFE3E0E3C0278007800F000F0 00F000F000F000F000780078003C013E0F1FFF0FFE03F010147E9314>I<000780000780 00078000078000078000078000078000078000078000078000078000078007C7800FF780 1FFF803E1F807C0780780780F80780F00780F00780F00780F00780F00780F00780F00780 780780780F803E1F801FFF800FF78007C78011207E9F17>I<03F0000FFC001FFE003E1F 003C0700780700700380FFFF80FFFF80FFFF80F00000F00000F000007000007800003C01 003E07001FFF0007FE0001F80011147F9314>I<007E01FE03FE078007000F000F000F00 0F000F000F000F00FFF0FFF0FFF00F000F000F000F000F000F000F000F000F000F000F00 0F000F000F000F000F000F000F20809F0E>I<03E0F00FFFF01FFFF03E3E003C1E00780F 00780F00780F00780F00780F003C1E003E3E001FFC003FF80033E0003000003800003FFE 003FFF801FFFC03FFFE07803F0F000F0F000F0F000F0F801F07E07E03FFFC00FFF0003FC 00141E7F9317>III<01E001E001E001E00000000000000000000000000000000001E0 01E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E001E0 01E001E001E001E001E001E0C3C0FFC0FF803F000B29839F0C>IIIII<01F80007FE001FFF803F0FC03C03C07801E07801E0 F000F0F000F0F000F0F000F0F000F0F000F07801E07801E03C03C03F0FC01FFF8007FE00 01F80014147F9317>II<03C7800F F7801FFF803E1F807C0F80780780780780F00780F00780F00780F00780F00780F00780F8 07807807807C0F803E1F801FFF800FF78007C78000078000078000078000078000078000 0780000780000780000780111D7E9317>II<07F01FFC3FFC78 0C7800780078007C003FC01FF00FF803F8007C003C003CC03CF07CFFF87FF00FC00E147F 9311>I<1E001E001E001E001E001E00FFF0FFF0FFF01E001E001E001E001E001E001E00 1E001E001E001E001E001E001E201FF00FF007C00C1A7F9910>IIII<7801E07C03C03E07801E0F000F0F00079E0003FC0003F800 01F80000F00001F00001F80003FC00079E000F0F000E0F001E07803C03C07801E0F801F0 1414809315>II<7FFF7FFF7FFF00 3E003C007800F800F001E003E007C007800F001F001E003C007C00FFFFFFFFFFFF10147F 9314>I E /Fw 32 123 df<007E01C007000E001C003C003800780078007FF0F000F000 F000700070007000300018000C1807E00F147E9312>15 D<70F8F8F87005057C840D>58 D<70F8FCFC74040404080810102040060E7C840D>I<000001C00000078000001E000000 78000001E00000078000000E00000038000000F0000003C000000F0000003C000000F000 0000F00000003C0000000F00000003C0000000F0000000380000000E0000000780000001 E0000000780000001E0000000780000001C01A1A7C9723>I<0001000300030006000600 06000C000C000C00180018001800300030003000600060006000C000C000C00180018001 800300030003000600060006000C000C000C001800180018003000300030006000600060 00C000C000C000102D7DA117>II<000002000000060000000E0000000E0000001E 0000001F0000002F0000002F0000004F0000008F0000008F0000010F0000010F0000020F 0000040F0000040F0000080F8000080780001007800020078000200780007FFF80004007 8000800780018007800100078002000780020007C0040003C00C0003C01E0007C0FF807F FC1E207E9F22>65 D<0FFFFFFC1E03C0381803C0181003C0082003C00820078008600780 084007800840078008800F0010000F0000000F0000000F0000001E0000001E0000001E00 00001E0000003C0000003C0000003C0000003C0000007800000078000000780000007800 0000F0000000F0000000F0000000F0000001F000007FFFC0001E1F7F9E1B>84 D<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E10F01C20F01C20703C20705C40308C400F078014147E9318>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207E9F14>I< 0000780003F80000700000700000700000700000E00000E00000E00000E00001C00001C0 00F1C00389C00707800E03801C03803C0380380700780700780700780700F00E00F00E00 F00E00F00E10F01C20F01C20703C20705C40308C400F078015207E9F18>100 D<007C01C207010E011C013C013802780C7BF07C00F000F000F000F00070007001700230 04183807C010147E9315>I<00007C0000CE00019E00039E00030C000700000700000700 000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C00001C00001C00 00380000380000380000380000380000700000700000700000700000700000E00000E000 00E00000E00000C00001C000318000798000F300006200003C000017297E9F16>I<001E 3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E01C01E01C01E01 C01E01C01E03801E03800E07800E0B8006170001E700000700000700000E00000E00300E 00781C00F038006070003FC000151D809316>I<01E0000FE00001C00001C00001C00001 C000038000038000038000038000070000070000071F000761800E80C00F00C00E00E00E 00E01C01C01C01C01C01C01C01C0380380380380380380380704700708700E08700E1070 0610E006206003C016207E9F1A>I<00E001E001E000C000000000000000000000000000 000E00130023804380438043808700070007000E000E001C001C001C2038403840384038 8019000E000B1F7E9E10>I<0000C00001E00001E00001C0000000000000000000000000 000000000000000000001E00006300004380008380010380010380020700000700000700 000700000E00000E00000E00000E00001C00001C00001C00001C00003800003800003800 00380000700000700030700078E000F1C0006380003E00001328819E13>I<01E0000FE0 0001C00001C00001C00001C0000380000380000380000380000700000700000701E00706 100E08700E10F00E20F00E40601C80001D00001E00001FC000387000383800383800381C 20703840703840703840701880E01880600F0014207E9F18>I<03C01FC0038003800380 038007000700070007000E000E000E000E001C001C001C001C0038003800380038007000 700070007100E200E200E200E200640038000A207E9F0E>I<1E07C07C00231861860023 A032030043C0340300438038038043803803808700700700070070070007007007000700 7007000E00E00E000E00E00E000E00E00E000E00E01C101C01C01C201C01C038201C01C0 38401C01C0184038038018801801800F0024147E9328>I<1E07802318C023A06043C070 4380704380708700E00700E00700E00700E00E01C00E01C00E01C00E03821C03841C0704 1C07081C03083803101801E017147E931B>I<007C0001C3000301800E01C01E01C01C01 E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780F00700700F00700E003018 0018700007C00013147E9316>I<03C1E004621804741C08781C08701E08701E10E01E00 E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380700380E003C1C007238007 1E000700000700000E00000E00000E00000E00001C00001C0000FFC000171D819317>I< 1E1E0023210023C38043C7804387804383008700000700000700000700000E00000E0000 0E00000E00001C00001C00001C00001C000038000018000011147E9315>114 D<007C018203010603060706060E00078007F803FC01FE001F00077007F006F006E00440 0820301FC010147E9315>I<00C000E001C001C001C001C003800380FFF8038007000700 070007000E000E000E000E001C001C001C001C10382038203820384018800F000D1C7F9B 10>I<0F00601180702180E021C0E041C0E04380E08381C00701C00701C00701C00E0380 0E03800E03800E03840E07080C07080C07080E0F1006131003E1E016147E931A>I<0F01 801183C02183E021C1E041C0E04380608380400700400700400700400E00800E00800E00 800E01000E01000C02000E04000E040006180001E00013147E9316>I<0F006060118070 F02180E0F821C0E07841C0E0384380E0188381C0100701C0100701C0100701C0100E0380 200E0380200E0380200E0380400E0380400E0380800E078080060781000709860001F078 001D147E9321>I<03C1C00C62201034701038F02038F020386040700000700000700000 700000E00000E00000E00000E02061C040F1C040F1C080E2C080446300383C0014147E93 1A>I<0F00601180702180E021C0E041C0E04380E08381C00701C00701C00701C00E0380 0E03800E03800E03800E07000C07000C07000E0F00061E0003EE00000E00000E00001C00 78180078380070700060600021C0001F0000141D7E9316>I<01E02003F04007F8C00C1F 8008010000020000040000080000100000600000C0000100000200000400800801001003 003F060061FC0040F80080700013147E9315>I E /Fx 54 122 df<000FF07F00007FF9 FF8000F83FC7C001E07F8FC003E07F0FC007C07F0FC007C03F078007C01F000007C01F00 0007C01F000007C01F000007C01F0000FFFFFFF800FFFFFFF80007C01F000007C01F0000 07C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007 C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F00003FF8 FFF0003FF8FFF0002220809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C00 07C07C0007C07C0007C0380007C0000007C0000007C0000007C1FE00FFFFFE00FFFFFE00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E003FF9FFC03FF9FFC0 1A20809F1D>I<000FF03FFC00007FFDFFFC0000F83FE0FC0001E03F81FC0003E07F81FC 0007C07F00FC0007C03F00FC0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C00FFFFFFFFFC00FFFFFFFFFC0007C01F007C0007C01F007C0007C01F007C 0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C003FF8FFE3FF803FF8FFE3FF802920809F2C>15 D<387CFEFFFF7F3B0303 06060E0C18702008107C860F>44 DI<387CFEFEFE7C 3807077C860F>I<01FC0007FF001F07C01E03C03E03E07C01F07C01F07C01F0FC01F8FC 01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F87C 01F07C01F07C01F03E03E01E03C01F8FC007FF0001FC00151D7E9C1A>48 D<00E00001E0000FE000FFE000F3E00003E00003E00003E00003E00003E00003E00003E0 0003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E0 0003E00003E00003E000FFFF80FFFF80111D7C9C1A>I<07F0001FFE00383F007C1F80FE 0FC0FE0FC0FE0FE0FE07E07C07E03807E0000FE0000FC0000FC0001F80001F00003E0000 780000F00000E00001C0000380600700600E00601C00E01FFFC03FFFC07FFFC0FFFFC0FF FFC0131D7D9C1A>I<01FC0007FF000E0F801E0FC03F07E03F07E03F07E03F07E01E0FC0 000FC0000F80001F0001FC0001FC00000F800007C00003E00003F00003F83803F87C03F8 FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF8003FC00151D7E9C1A>I<0001C00003 C00007C00007C0000FC0001FC0003BC00073C00063C000C3C00183C00383C00703C00E03 C00C03C01803C03803C07003C0E003C0FFFFFEFFFFFE0007C00007C00007C00007C00007 C00007C000FFFE00FFFE171D7F9C1A>I<3803803FFF803FFF003FFE003FFC003FF0003F 800030000030000030000030000033F80037FE003C1F00380F801007C00007C00007E000 07E07807E0FC07E0FC07E0FC07E0FC07C0780FC0600F80381F001FFC0007F000131D7D9C 1A>I<003F0001FFC007E0E00F81E01F03F01E03F03E03F07C03F07C01E07C0000FC1000 FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8FC01F8FC01F87C01F87C01F87C01F8 3C01F03E01F01E03E00F07C007FF8001FE00151D7E9C1A>I<6000007FFFF87FFFF87FFF F07FFFE07FFFC0E00180C00300C00300C00600000C000018000038000038000078000070 0000F00000F00001F00001F00001F00001F00003F00003F00003F00003F00003F00003F0 0001E00000C000151E7D9D1A>I<01FC0007FF000F07801E03C01C01E03C01E03C01E03E 01E03F01E03FC3C01FE3801FFF000FFE0007FF8007FFC01FFFE03C3FF0780FF07803F8F0 01F8F000F8F00078F00078F000707800707C00E03E03C00FFF8003FC00151D7E9C1A>I< 01FC000FFF001F07803E03C07C03E07C01E0FC01F0FC01F0FC01F0FC01F8FC01F8FC01F8 FC01F87C03F87C03F83E05F81FFDF807F9F80041F80001F03C01F07E01F07E03E07E03E0 7E07C03C0780381F001FFC0007F000151D7E9C1A>I<387CFEFEFE7C3800000000000038 7CFEFEFE7C3807147C930F>I<0000E000000000E000000001F000000001F000000001F0 00000003F800000003F800000006FC00000006FC0000000EFE0000000C7E0000000C7E00 0000183F000000183F000000303F800000301F800000701FC00000600FC00000600FC000 00C007E00000FFFFE00001FFFFF000018003F000018003F000030001F800030001F80006 0001FC00060000FC000E0000FE00FFE00FFFE0FFE00FFFE0231F7E9E28>65 DI<0007FC02003FFF0E00FE03DE 03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E0000067E000006 FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E0000007E000006 7F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000FE01C0003FFF80 0007FC001F1F7D9E26>I69 DI<0007FC0200003FFF0E0000FE 03DE0003F000FE0007E0003E000FC0001E001F80001E003F00000E003F00000E007F0000 06007E000006007E00000600FE00000000FE00000000FE00000000FE00000000FE000000 00FE003FFFE0FE003FFFE07E00007E007E00007E007F00007E003F00007E003F00007E00 1F80007E000FC0007E0007E0007E0003F000FE0000FE01FE00003FFF8E000007FC060023 1F7D9E29>I73 D76 DI 80 D<03FC080FFF381E03F83800F8700078700038F00038F00018F00018F80000FC0000 7FC0007FFE003FFF801FFFE00FFFF007FFF000FFF80007F80000FC00007C00003CC0003C C0003CC0003CE00038E00078F80070FE01E0E7FFC081FF00161F7D9E1D>83 D<7FFFFFFC7FFFFFFC7C07E07C7007E01C6007E00C6007E00CE007E00EC007E006C007E0 06C007E006C007E0060007E0000007E0000007E0000007E0000007E0000007E0000007E0 000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0 000007E00003FFFFC003FFFFC01F1E7E9D24>I<07FC001FFF003F0F803F07C03F03E03F 03E00C03E00003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F803E0FC05E07E 0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC 0000FC0000FC0000FC0000FC00007C00007E00003E00603F00C01F81C007FF0001FC0013 147E9317>I<0007F80007F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F801F8F80FFEF81F83F83E01F87E00F87C00F87C00F8FC00F8FC00F8FC00F8 FC00F8FC00F8FC00F87C00F87C00F87E00F83E01F81F07F80FFEFF03F8FF18207E9F1D> I<01FE0007FF800F83C01E01E03E00F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC00 00FC00007C00007C00003E00181E00180F807007FFE000FF8015147F9318>I<001F8000 FFC001F3E003E7E003C7E007C7E007C3C007C00007C00007C00007C00007C000FFFC00FF FC0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007 C00007C00007C00007C00007C0003FFC003FFC0013207F9F10>I<01FC3C07FFFE0F079E 1E03DE3E03E03E03E03E03E03E03E03E03E01E03C00F07800FFF0009FC00180000180000 1C00001FFF800FFFF007FFF81FFFFC3C007C70003EF0001EF0001EF0001E78003C78003C 3F01F80FFFE001FF00171E7F931A>II<1C003E007F007F007F003E001C000000000000000000000000 00FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F 00FFE0FFE00B217EA00E>I<0038007C00FE00FE00FE007C003800000000000000000000 000001FE01FE003E003E003E003E003E003E003E003E003E003E003E003E003E003E003E 003E003E003E003E003E303E783EFC3CFC7C78783FF01FC00F2A83A010>IIIII<01FF0007FFC01F83F03E00F83E00F87C00 7C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C007C7C007C3E00F83E00F81F83 F007FFC001FF0017147F931A>II< 01F81807FE381F87783F01F83E01F87E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8 FC00F87C00F87E00F87E00F83F01F81F87F80FFEF803F8F80000F80000F80000F80000F8 0000F80000F80000F80007FF0007FF181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07FF83F FC1FFE03FE001FC007C007E007F006F81EFFFCC7F010147E9315>I<0180018001800380 0380038007800F803F80FFFCFFFC0F800F800F800F800F800F800F800F800F800F800F86 0F860F860F860F8607CC03F801F00F1D7F9C14>IIIIII E /Fy 37 122 df<00000007FF800000000001FFFFF0000000000FFFFFFC000000003FFFFFFE00 000000FFFC00FF00000001FFC0003F80000007FF00007FC000000FFE0001FFC000001FFC 0001FFE000001FF80003FFE000003FF00003FFE000003FF00003FFE000007FE00003FFE0 00007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00001FFC000007FE0 0000FF8000007FE000003E0000007FE00000000000007FE00000000000007FE000000000 00007FE00000000000007FE00000000000007FE00000000000007FE0003FFFF000FFFFFF FFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF0 00007FF00000FFF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFF E03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC007FFE00FFFE00FFFF00FFFF 00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E780000780000780000F00000F 00000F00000F00001E00001E00003C00003C0000780000F80000F00001F00003E00007C0 000F80001F00001E00000C00001124788F21>44 D<0000000003E0000000000000000000 07F000000000000000000007F00000000000000000000FF80000000000000000000FF800 00000000000000000FF80000000000000000001FFC0000000000000000001FFC00000000 00000000003FFE0000000000000000003FFE0000000000000000007FFF00000000000000 00007FFF0000000000000000007FFF000000000000000000FFFF800000000000000000FF FF800000000000000001FFFFC00000000000000001FFFFC00000000000000001FFFFC000 00000000000003FFFFE00000000000000003EFFFE00000000000000007EFFFF000000000 00000007CFFFF00000000000000007C7FFF0000000000000000FC7FFF800000000000000 0F83FFF8000000000000001F83FFFC000000000000001F03FFFC000000000000001F01FF FC000000000000003F01FFFE000000000000003E00FFFE000000000000007E00FFFF0000 00000000007C007FFF00000000000000FC007FFF80000000000000F8007FFF8000000000 0000F8003FFF80000000000001F8003FFFC0000000000001F0001FFFC0000000000003F0 001FFFE0000000000003E0001FFFE0000000000003E0000FFFE0000000000007E0000FFF F0000000000007C00007FFF000000000000FC00007FFF800000000000F800007FFF80000 0000000F800003FFF800000000001F800003FFFC00000000001F000001FFFC0000000000 3FFFFFFFFFFE00000000003FFFFFFFFFFE00000000007FFFFFFFFFFF00000000007FFFFF FFFFFF00000000007FFFFFFFFFFF0000000000FC0000007FFF8000000000F80000003FFF 8000000001F80000003FFFC000000001F00000003FFFC000000001F00000001FFFC00000 0003F00000001FFFE000000003E00000000FFFE000000007E00000000FFFF000000007C0 0000000FFFF000000007C000000007FFF00000000FC000000007FFF80000000F80000000 03FFF80000001F8000000003FFFC0000001F0000000003FFFC0000007FC000000001FFFC 0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FF FFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF8051487CC75A>65 DI<00 0000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E000003FFFFFFFE00FE00 000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001FFFC000003FFFE00 07FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003FFF000000001FFE00 7FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FFF00000000003FE03 FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFFC00000000000FE0F FFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF8000000000007E3F FF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF0000000000003E7F FF000000000000007FFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FF FF00000000000000FFFF00000000000000FFFF000000000000007FFF000000000000007F FF000000000000007FFF000000000000007FFF8000000000003E3FFF8000000000003E3F FF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFFC000000000003E0F FFC000000000007C0FFFC000000000007C07FFE000000000007C07FFE00000000000F803 FFF00000000000F803FFF00000000001F801FFF80000000001F000FFFC0000000003E000 7FFE0000000007E0003FFF000000000FC0001FFF800000001F80000FFFC00000003F0000 07FFF0000000FE000001FFFC000001FC000000FFFF80000FF80000003FFFF8007FF00000 000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000000FFFFFE0000000 0000003FFE000000474979C756>I69 DI<000000003FFE 00000E000000000FFFFFC0001E000000007FFFFFF8003E00000003FFFFFFFE00FE000000 0FFFFFFFFF81FE0000003FFFF800FFC3FE000000FFFF80000FF7FE000001FFFC000003FF FE000007FFF0000001FFFE00000FFFC00000007FFE00001FFF800000003FFE00003FFF00 0000001FFE00007FFE000000000FFE0000FFFC0000000007FE0001FFF80000000007FE00 03FFF00000000003FE0003FFF00000000001FE0007FFE00000000001FE0007FFE0000000 0000FE000FFFC00000000000FE000FFFC000000000007E001FFFC000000000007E001FFF 8000000000007E003FFF8000000000007E003FFF8000000000003E003FFF800000000000 3E007FFF8000000000003E007FFF0000000000003E007FFF00000000000000007FFF0000 000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000 FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF00000000 00000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF 0000000000000000FFFF0000000000000000FFFF0000007FFFFFFFFE7FFF0000007FFFFF FFFE7FFF0000007FFFFFFFFE7FFF8000007FFFFFFFFE7FFF8000007FFFFFFFFE3FFF8000 000000FFFE003FFF8000000000FFFE003FFF8000000000FFFE001FFF8000000000FFFE00 1FFFC000000000FFFE000FFFC000000000FFFE000FFFC000000000FFFE0007FFE0000000 00FFFE0007FFE000000000FFFE0003FFF000000000FFFE0003FFF800000000FFFE0001FF F800000000FFFE0000FFFC00000000FFFE00007FFE00000000FFFE00003FFF00000000FF FE00001FFF80000000FFFE00000FFFE0000001FFFE000007FFF0000003FFFE000001FFFC 000007FFFE000000FFFF80001FFFFE0000003FFFFC00FFC7FE0000000FFFFFFFFF83FE00 000003FFFFFFFE00FE000000007FFFFFF8003E000000000FFFFFE0000E00000000003FFE 000000004F4979C75D>I73 D76 DI80 D<00007FF00007000007FFFF000F00001FFFFFC01F00007FFFFFF03F0000 FFFFFFFC7F0003FFC00FFEFF0007FE0000FFFF0007FC00003FFF000FF000001FFF001FF0 000007FF001FE0000003FF003FE0000001FF003FC0000001FF007FC0000000FF007FC000 00007F007FC00000007F00FFC00000003F00FFC00000003F00FFC00000003F00FFE00000 003F00FFE00000001F00FFF00000001F00FFF80000001F00FFFC0000001F00FFFF000000 00007FFFC0000000007FFFFE000000007FFFFFE00000003FFFFFFF0000003FFFFFFFF000 001FFFFFFFFC00001FFFFFFFFF00000FFFFFFFFFC00007FFFFFFFFE00007FFFFFFFFF000 03FFFFFFFFF80000FFFFFFFFFC00007FFFFFFFFE00003FFFFFFFFE00000FFFFFFFFF0000 01FFFFFFFF0000003FFFFFFF80000001FFFFFF800000000FFFFFC000000000FFFFC00000 00001FFFC0000000000FFFE00000000007FFE00000000003FFE07800000001FFE0F80000 0000FFE0F800000000FFE0F8000000007FE0F8000000007FE0F8000000007FE0FC000000 007FE0FC000000007FC0FC000000007FC0FE000000007FC0FE000000007F80FF00000000 FF80FF80000000FF80FFC0000000FF00FFE0000001FE00FFF8000003FE00FFFE000007FC 00FFFF80000FF800FFFFFC007FF000FE3FFFFFFFE000FC0FFFFFFF8000F803FFFFFF0000 F0007FFFF80000E00003FFC00000334979C742>83 D<3FFFFFFFFFFFFFFFFF003FFFFFFF FFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF00 3FFF0003FFF8003FFF007FF80003FFF80007FF807FE00003FFF80001FF807FC00003FFF8 0000FF807F800003FFF800007F807F000003FFF800003F807F000003FFF800003F807E00 0003FFF800001F807E000003FFF800001F807E000003FFF800000F807C000003FFF80000 0F807C000003FFF800000F807C000003FFF800000F807C000003FFF800000F80FC000003 FFF800000FC0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007C0 F8000003FFF8000007C0F8000003FFF8000007C000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF8000000000003FFFFFFFFFFF800000003 FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF8 00004A467CC553>I<0007FFFC000000007FFFFFC0000001FFFFFFF8000003FFFFFFFE00 0007FE001FFF000007FF0003FFC0000FFF8001FFE0000FFF8000FFF0000FFF80007FF000 0FFF80007FF8000FFF80007FF80007FF00003FFC0007FF00003FFC0003FE00003FFC0000 F800003FFC00000000003FFC00000000003FFC00000000003FFC00000000003FFC000000 07FFFFFC000000FFFFFFFC000007FFFFFFFC00003FFFE03FFC0000FFFE003FFC0003FFF0 003FFC0007FFC0003FFC000FFF00003FFC001FFE00003FFC003FFC00003FFC007FF80000 3FFC007FF800003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000003F FC00FFF000003FFC00FFF000007FFC007FF80000FFFC007FF80001EFFC003FFC0003EFFC 003FFF0007CFFF000FFFC03F8FFFF807FFFFFF07FFFC01FFFFFC03FFFC007FFFF001FFFC 0003FF80007FF8362E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC00000 000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC00FFC00000001FFC07FFFC0000001FFC3FFFFF0000001FFCFFFFF FC000001FFDFF00FFF000001FFFF8003FF800001FFFE0001FFC00001FFF800007FE00001 FFF000007FF00001FFE000003FF80001FFE000001FFC0001FFE000001FFC0001FFE00000 1FFE0001FFE000001FFE0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001 FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000000FFF0001FFE00000 0FFF0001FFE000001FFE0001FFE000001FFE0001FFE000001FFC0001FFE000001FFC0001 FFE000003FF80001FFF000003FF80001FFF800007FF00001FFFC0000FFE00001FFFE0001 FFC00001FFBF0007FF800001FF1FE01FFE000001FE0FFFFFFC000001FC03FFFFF0000001 F800FFFF80000001F0001FF800000039487CC742>I<00001FFFC0000000FFFFF8000007 FFFFFE00001FFFFFFF80007FFC00FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF 0003FFE00FFE0003FFE00FFE0003FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC 00003E007FF8000000007FF8000000007FF800000000FFF800000000FFF800000000FFF8 00000000FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF8 00000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC000000003FFC 000000001FFC000000F81FFE000000F80FFE000000F80FFF000001F007FF800003F003FF C00007E001FFE0000FC000FFF0001F80007FFE00FF00001FFFFFFE000007FFFFF8000000 FFFFE00000001FFE00002D2E7CAD35>I<00000000007FC00000000000FFFFC000000000 00FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC00000000FFC01FFC0000000FFFF81FFC0000007FFFFE1FFC000001FFF FFF9FFC000007FFC03FFFFC00000FFF0007FFFC00001FFC0001FFFC00003FF80000FFFC0 0007FF000007FFC0000FFE000003FFC0000FFE000003FFC0001FFC000003FFC0001FFC00 0003FFC0003FFC000003FFC0003FFC000003FFC0007FF8000003FFC0007FF8000003FFC0 007FF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF800 0003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC0 00FFF8000003FFC000FFF8000003FFC0007FF8000003FFC0007FF8000003FFC0007FF800 0003FFC0003FF8000003FFC0003FFC000003FFC0003FFC000003FFC0001FFC000003FFC0 001FFC000003FFC0000FFE000007FFC00007FF00000FFFC00003FF00001FFFC00001FFC0 003FFFC00000FFE000FFFFE000007FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FFFF 800001FFFF03FFFF8000001FF803FFFF8039487CC742>I<00001FFE00000001FFFFE000 0007FFFFF800001FFFFFFE00007FFC07FF0000FFE001FF8001FFC0007FC003FF80003FE0 07FF00003FF00FFE00001FF01FFE00000FF81FFC00000FF83FFC00000FFC3FFC000007FC 7FFC000007FC7FF8000007FC7FF8000007FE7FF8000007FEFFF8000007FEFFF8000007FE FFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFCFFF800000000FFF800000000 FFF800000000FFF8000000007FF8000000007FF8000000007FFC000000003FFC00000000 3FFC000000003FFC0000001C1FFE0000003E0FFE0000003E07FF0000007E07FF000000FC 03FF800001F801FFC00003F0007FF0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE00 0000FFFFF80000000FFF80002F2E7DAD36>I<000000FFC000000007FFF80000003FFFFC 000000FFFFFF000001FFC1FF000007FF03FF80000FFC03FF80000FF807FFC0001FF807FF C0003FF007FFC0003FF007FFC0003FE003FF80007FE003FF80007FE001FF00007FE000FE 00007FE0003800007FE0000000007FE0000000007FE0000000007FE0000000007FE00000 00007FE0000000007FE0000000007FE0000000007FE0000000007FE0000000FFFFFFFE00 00FFFFFFFE0000FFFFFFFE0000FFFFFFFE0000FFFFFFFE0000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF00000 00007FF0000000007FF0000000007FF0000000007FF00000003FFFFFF800003FFFFFF800 003FFFFFF800003FFFFFF800003FFFFFF800002A487DC724>I<00000000001F8000007F F000FFE00007FFFF03FFF0001FFFFFC7FFF0007FFFFFFFC7F800FFE03FFE0FF801FF800F FC0FF803FF0007FE0FF807FE0003FF07F007FE0003FF07F00FFC0001FF81C00FFC0001FF 80000FFC0001FF80001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0 001FFC0001FFC0001FFC0001FFC0000FFC0001FF80000FFC0001FF80000FFC0001FF8000 07FE0003FF000007FE0003FF000003FF0007FE000001FF800FFC000000FFE03FF8000001 FFFFFFF0000001DFFFFFC0000003C7FFFF00000003C07FF000000007C0000000000007C0 000000000007C0000000000007C0000000000007E0000000000007F0000000000007F800 0000000007FFFFFFF0000007FFFFFFFF000003FFFFFFFFE00003FFFFFFFFF80001FFFFFF FFFE0001FFFFFFFFFF0000FFFFFFFFFF80007FFFFFFFFF8003FFFFFFFFFFC00FFFFFFFFF FFC01FF800001FFFE03FE0000001FFE07FC00000007FF07FC00000003FF0FF800000001F F0FF800000001FF0FF800000001FF0FF800000001FF0FF800000001FF07FC00000003FE0 7FC00000003FE03FE00000007FC03FF0000000FFC01FFC000003FF800FFF00000FFF0003 FFF000FFFC0000FFFFFFFFF000003FFFFFFFC0000007FFFFFE000000003FFFC000003544 7DAE3B>I<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC000000000 00FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 01FFC0000001FFC00FFFF8000001FFC03FFFFE000001FFC0FFFFFF000001FFC1FC07FF80 0001FFC3E003FFC00001FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC 0000FFE00001FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF0 0001FFF00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 00FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFF C07FFFFFE03B487CC742>I<00FC0001FE0003FF0007FF800FFFC01FFFE01FFFE01FFFE0 1FFFE01FFFE01FFFE00FFFC007FF8003FF0001FE0000FC00000000000000000000000000 000000000000000000000000000000000000000000007FC0FFFFC0FFFFC0FFFFC0FFFFC0 FFFFC003FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 01FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 01FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0 FFFFFFFFFFFFFFFFFFFFFFFFFFFFFF18497CC820>I<007FC000000000FFFFC000000000 FFFFC000000000FFFFC000000000FFFFC000000000FFFFC00000000003FFC00000000001 FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FF C00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC0 0000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000 00000001FFC00000000001FFC00000000001FFC00000000001FFC0007FFFF801FFC0007F FFF801FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0000FFE0001FFC00007F0 0001FFC0000FE00001FFC0003FC00001FFC0007F800001FFC000FE000001FFC001FC0000 01FFC007F8000001FFC00FF0000001FFC01FC0000001FFC03F80000001FFC0FF00000001 FFC1FE00000001FFC3FF00000001FFCFFF80000001FFDFFFC0000001FFFFFFC0000001FF FFFFE0000001FFFFFFF0000001FFFCFFF8000001FFF87FFC000001FFE03FFC000001FFC0 1FFE000001FFC01FFF000001FFC00FFF800001FFC007FFC00001FFC003FFC00001FFC001 FFE00001FFC001FFF00001FFC000FFF80001FFC0007FFC0001FFC0003FFC0001FFC0001F FE0001FFC0000FFF0001FFC0000FFF8001FFC0000FFFC0FFFFFF807FFFFFFFFFFF807FFF FFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FFFFC000FFFFC000FFFFC00003FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC000FFFFFF80FFFFFF80FFFFFF80FFFFFF80FFFFFF 8019487CC720>I<007FC001FFC00000FFE00000FFFFC00FFFF80007FFFC0000FFFFC03F FFFE001FFFFF0000FFFFC0FFFFFF007FFFFF8000FFFFC1FC07FF80FE03FFC000FFFFC3E0 03FFC1F001FFE00003FFC7C001FFC3E000FFE00001FFCF0001FFE78000FFF00001FFDE00 00FFEF00007FF00001FFDC0000FFEE00007FF00001FFFC0000FFFE00007FF80001FFF800 00FFFC00007FF80001FFF00000FFF800007FF80001FFF00000FFF800007FF80001FFF000 00FFF800007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE000 00FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800FFFFFFC0 7FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC0 7FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF05C2E7CAD63>I<007FC001FFC00000FF FFC00FFFF80000FFFFC03FFFFE0000FFFFC0FFFFFF0000FFFFC1FC07FF8000FFFFC3E003 FFC00003FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001 FFFC0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF00001FFF00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFFC07F FFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B 2E7CAD42>I<00000FFF0000000000FFFFF000000007FFFFFE0000001FFFFFFF8000003F FC03FFC00000FFE0007FF00001FF80001FF80003FF00000FFC0007FE000007FE000FFE00 0007FF000FFC000003FF001FFC000003FF803FFC000003FFC03FF8000001FFC03FF80000 01FFC07FF8000001FFE07FF8000001FFE07FF8000001FFE0FFF8000001FFF0FFF8000001 FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FF F0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF07FF8000001FFE07FF8000001FFE0 7FF8000001FFE07FF8000001FFE03FFC000003FFC03FFC000003FFC01FFC000003FF801F FE000007FF800FFE000007FF0007FF00000FFE0003FF80001FFC0001FFC0003FF80000FF E0007FF000007FFC03FFE000001FFFFFFF80000007FFFFFE00000000FFFFF0000000000F FF000000342E7DAD3B>I<007FC00FFC000000FFFFC07FFFC00000FFFFC3FFFFF00000FF FFCFFFFFFC0000FFFFDFF01FFF0000FFFFFF8007FF800003FFFE0001FFC00001FFF80000 FFE00001FFF00000FFF00001FFE000007FF80001FFE000003FFC0001FFE000003FFC0001 FFE000003FFE0001FFE000001FFE0001FFE000001FFF0001FFE000001FFF0001FFE00000 1FFF0001FFE000000FFF0001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF0001FFE000001FFF0001 FFE000001FFF0001FFE000001FFE0001FFE000001FFE0001FFE000003FFC0001FFE00000 3FFC0001FFE000007FF80001FFF000007FF80001FFF80000FFF00001FFFC0001FFE00001 FFFE0003FFC00001FFFF0007FF800001FFFFE03FFE000001FFEFFFFFFC000001FFE3FFFF F0000001FFE0FFFF80000001FFE01FF800000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE00000 00000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE00000000000FFFFFFC000 000000FFFFFFC000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC00000000039 427CAD42>I<00FF803F8000FFFF80FFF000FFFF83FFFC00FFFF87FFFE00FFFF8FC3FF00 FFFF8F07FF0003FF9E0FFF8001FFBC0FFF8001FFB80FFF8001FFF80FFF8001FFF00FFF80 01FFF007FF0001FFF007FF0001FFE003FE0001FFE000F80001FFE000000001FFE0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFFFFE00000292E7CAD31> 114 D<000FFF00E0007FFFF3E001FFFFFFE007FFFFFFE00FF800FFE01FC0001FE03F8000 0FE03F000007E07F000003E07F000003E0FF000003E0FF000003E0FF800003E0FFC00000 00FFF0000000FFFE000000FFFFF800007FFFFFC0007FFFFFF0003FFFFFFC001FFFFFFF00 0FFFFFFF8007FFFFFFC003FFFFFFE000FFFFFFF0003FFFFFF00003FFFFF800001FFFF800 0000FFFC0000001FFC7800000FFCF8000007FCF8000003FCFC000003FCFC000003FCFE00 0003F8FE000003F8FF000003F8FF800007F0FFC0000FF0FFF0001FE0FFFC00FFC0FFFFFF FF80FC7FFFFE00F81FFFF800E003FF8000262E7CAD2F>I<0001F000000001F000000001 F000000001F000000001F000000001F000000003F000000003F000000003F000000007F0 00000007F000000007F00000000FF00000000FF00000001FF00000003FF00000003FF000 00007FF0000001FFF0000003FFF000000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC0 FFFFFFFFC000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000 FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0 000000FFF0000000FFF0000000FFF001F000FFF001F000FFF001F000FFF001F000FFF001 F000FFF001F000FFF001F000FFF001F000FFF001F0007FF001E0007FF803E0003FF803E0 003FFC07C0001FFE0F80000FFFFF800007FFFE000001FFFC0000001FF00024427EC12E> I<007FE000003FF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0 007FFFF000FFFFE0007FFFF00003FFE00001FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00001FFF00001FFE00001FFF00001FFE00001FFF0 0001FFE00003FFF00000FFE00007FFF00000FFE0000F7FF000007FE0001F7FF000007FF0 003E7FF800003FFC00FC7FFFE0001FFFFFF87FFFE00007FFFFE07FFFE00001FFFF807FFF E000003FFE007FFFE03B2E7CAD42>III<7FFFFF801FFFFF007FFFFF801FFFFF00 7FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF00007FF80001FF0000007FFC00 01FC0000003FFE0003F80000001FFF0007F00000000FFF0007E000000007FF800FC00000 0003FFC01F8000000003FFE03F8000000001FFF07F0000000000FFF8FE00000000007FF9 FC00000000003FFFF800000000003FFFF000000000001FFFE000000000000FFFC0000000 000007FFC0000000000003FFC0000000000001FFE0000000000001FFF0000000000001FF F8000000000003FFFC000000000003FFFE000000000007FFFE00000000000FEFFF000000 00001FCFFF80000000003F87FFC0000000007F03FFE000000000FE01FFE000000001FC00 FFF000000001F8007FF800000003F0007FFC00000007F0003FFE0000000FE0001FFF0000 001FC0000FFF0000003F800007FF800000FF800007FFC000FFFFF8003FFFFFC0FFFFF800 3FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC03A2E7EAD3F>I<7F FFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000 FFFF8000FFF000000FE00000FFF800000FC00000FFF800000FC000007FFC00000F800000 7FFC00001F8000003FFC00001F0000003FFE00003F0000001FFE00003E0000001FFF0000 7E0000000FFF00007C0000000FFF8000FC00000007FF8000F800000007FFC001F8000000 03FFC001F000000003FFE003F000000003FFE003F000000001FFF003E000000001FFF007 E000000000FFF007C000000000FFF80FC0000000007FF80F80000000007FFC1F80000000 003FFC1F00000000003FFE3F00000000001FFE3E00000000001FFF7E00000000000FFF7C 00000000000FFFFC00000000000FFFFC000000000007FFF8000000000007FFF800000000 0003FFF0000000000003FFF0000000000001FFE0000000000001FFE0000000000000FFC0 000000000000FFC00000000000007F800000000000007F800000000000003F0000000000 00003F000000000000003F000000000000003E000000000000007E000000000000007C00 000000000000FC000000001F8000F8000000003FC001F8000000007FE001F000000000FF F003F000000000FFF003E000000000FFF007E000000000FFF00FC000000000FFF01F8000 000000FFF03F80000000007FE07F00000000007F43FE00000000003FFFF800000000001F FFF0000000000007FFC0000000000001FE00000000000039427EAD3F>I E /Fz 10 58 df<1F00318060C04040C060C060C060C060C060C060C060C060404060C0 31801F000B107F8F0F>48 D<0C003C00CC000C000C000C000C000C000C000C000C000C00 0C000C000C00FF8009107E8F0F>I<1F00618040C08060C0600060006000C00180030006 000C00102020207FC0FFC00B107F8F0F>I<1F00218060C060C000C0008001800F000080 00400060C060C060804060801F000B107F8F0F>I<0300030007000F000B001300330023 004300C300FFE003000300030003001FE00B107F8F0F>I<20803F002C00200020002000 2F0030802040006000600060C06080C061801F000B107F8F0F>I<0780184030C060C060 00C000CF00F080E040C060C060C060406060C030801F000B107F8F0F>I<40007FE07FC0 8080808001000200040004000C0008000800180018001800180018000B117E900F>I<1F 00318060C060C060C071803F000F00338061C0C060C060C060404060801F000B107F8F0F >I<1F00318060C0C040C060C060C06040E021E01E600060004060C0608043003E000B10 7F8F0F>I E /FA 14 104 df0 D<70F8F8F87005057C8D0D>I<400004C0000C6000183000301800600C00C00601800303 0001860000CC0000780000300000300000780000CC000186000303000601800C00C01800 60300030600018C0000C40000416187A9623>I<01800180018001800180C183F18F399C 0FF003C003C00FF0399CF18FC1830180018001800180018010147D9417>I<0001FE0000 0007FF8000001E01E000007800780000E0001C000180000600030000030006000001800C 000000C00C000000C0180000006030000000303000000030300000003060000000186000 0000186000000018C00000000CC00000000CC00000000CC00000000CC00000000CC00000 000CC00000000CC00000000CC00000000C60000000186000000018600000001830000000 303000000030300000003018000000600C000000C00C000000C006000001800300000300 018000060000E0001C000078007800001E01E0000007FF80000001FE0000262B7DA02D> 13 D<03C00FF01C38300C60066006C003C003C003C003C003C00360066006300C1C380F F003C010127D9317>I<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF7FFE 7FFE3FFC1FF80FF003C010127D9317>I<000000C0000003C000000F0000003C000000F0 000003C00000070000001C00000078000001E00000078000001E00000078000000E00000 00780000001E0000000780000001E0000000780000001C0000000700000003C0000000F0 0000003C0000000F00000003C0000000C000000000000000000000000000000000000000 0000000000000000007FFFFF80FFFFFFC01A247C9C23>20 DI<003FF800 FFF803C0000700000C0000180000300000300000600000600000C00000C00000C00000FF FFF8FFFFF8C00000C00000C000006000006000003000003000001800000C000007000003 C00000FFF8003FF8151C7C981E>50 D<00000C00000C0000180000180000300000300000 600000600000C00000C0000180000180000180000300000300000600000600000C00000C 0000180000180000300000300000600000600000C00000C0000180000180000300000300 000600000600000600000C00000C0000180000180000300000300000600000600000C000 00400000162C7AA000>54 D<400002C00006C00006C00006C00006C00006C00006C00006 C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006 C0000660000C60000C3000181C00700F01E003FF8000FE00171C7D9A1E>91 D<000F0038006000E001C001C001C001C001C001C001C001C001C001C001C001C001C001 C001C0038007001E00F8001E000700038001C001C001C001C001C001C001C001C001C001 C001C001C001C001C001C000E000600038000F102D7DA117>102 DI E /FB 1 98 df<001800001800001800003C00003C00004E00004E00004E00008700008700 0187800103800103800201C00201C003FFC00400E00400E00800700800701800703C0078 FE01FF18177F961C>97 D E /FC 91 128 df<001F83E000F06E3001C078780380F87803 00F03007007000070070000700700007007000070070000700700007007000FFFFFF8007 007000070070000700700007007000070070000700700007007000070070000700700007 00700007007000070070000700700007007000070070000700700007007000070070007F E3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E007000007000007 0000070000070000070000FFFFE00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E07FC3FE17 20809F19>I<003FE000E0E001C1E00381E00700E00700E00700E00700E00700E00700E0 0700E00700E0FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E0 0700E00700E00700E00700E00700E00700E00700E00700E00700E07FE7FE1720809F19> I<001F81F80000F04F040001C07C06000380F80F000300F00F000700F00F000700700000 07007000000700700000070070000007007000000700700000FFFFFFFF00070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 0700070070070007007007007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C0 7C0F000380F80F000300F007000700700700070070070007007007000700700700070070 070007007007000700700700FFFFFFFF0007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 07007007000700700700070070070007007007000700700700070070070007007007007F E3FE3FF02420809F26>I22 D<7038F87CFC7EFC7E743A0402040204 020804080410081008201040200F0E7E9F17>34 D<000300C0000300C0000300C0000300 C00006018000060180000601800006018000060180000C0300000C0300000C0300000C03 00000C0300001806007FFFFFFCFFFFFFFE00300C0000300C0000300C0000300C00006018 00006018000060180000601800FFFFFFFE7FFFFFFC00C030000180600001806000018060 0001806000018060000300C0000300C0000300C0000300C0000300C00006018000060180 00060180001F297D9F26>I<007800000084000001840000030200000702000007020000 0702000007020000070400000704000007080000070800000310000003A00FFC03C003E0 038001C001C0008001C0010003E0010004E0020008F00200187004003078080070380800 701C1000F01E1000F00E2000F0074000F003C0087003C0087801C010380670301C183860 07E00F801E227EA023>38 D<70F8FCFC74040404080810102040060E7C9F0D>I<002000 4000800100020006000C000C00180018003000300030007000600060006000E000E000E0 00E000E000E000E000E000E000E000E000E0006000600060007000300030003000180018 000C000C000600020001000080004000200B2E7DA112>I<800040002000100008000C00 060006000300030001800180018001C000C000C000C000E000E000E000E000E000E000E0 00E000E000E000E000E000C000C000C001C001800180018003000300060006000C000800 10002000400080000B2E7DA112>I<01800180018001800180C183F18F399C0FF003C003 C00FF0399CF18FC1830180018001800180018010147DA117>I<00060000000600000006 000000060000000600000006000000060000000600000006000000060000000600000006 0000000600000006000000060000FFFFFFF0FFFFFFF00006000000060000000600000006 000000060000000600000006000000060000000600000006000000060000000600000006 000000060000000600001C207D9A23>I<70F8FCFC74040404080810102040060E7C840D> II<70F8F8F87005057C840D>I<00010003000300060006000600 0C000C000C00180018001800300030003000600060006000C000C000C001800180018003 00030003000600060006000C000C000C00180018001800300030003000600060006000C0 00C000C000102D7DA117>I<03F0000E1C001C0E00180600380700700380700380700380 700380F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0 F003C0F003C07003807003807003807807803807001806001C0E000E1C0003F000121F7E 9D17>I<018003800F80F380038003800380038003800380038003800380038003800380 03800380038003800380038003800380038003800380038007C0FFFE0F1E7C9D17>I<03 F0000C1C00100E00200700400780800780F007C0F803C0F803C0F803C02007C00007C000 0780000780000F00000E00001C0000380000700000600000C0000180000300000600400C 00401800401000803FFF807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00 780F80780780780780380F80000F80000F00000F00000E00001C0000380003F000003C00 000E00000F000007800007800007C02007C0F807C0F807C0F807C0F00780400780400F00 200E001C3C0003F000121F7E9D17>I<000600000600000E00000E00001E00002E00002E 00004E00008E00008E00010E00020E00020E00040E00080E00080E00100E00200E00200E 00400E00C00E00FFFFF0000E00000E00000E00000E00000E00000E00000E0000FFE0141E 7F9D17>I<1803001FFE001FFC001FF8001FE00010000010000010000010000010000010 000011F000161C00180E001007001007800003800003800003C00003C00003C07003C0F0 03C0F003C0E00380400380400700200600100E000C380003E000121F7E9D17>I<007C00 0182000701000E03800C07801C0780380300380000780000700000700000F1F000F21C00 F40600F80700F80380F80380F003C0F003C0F003C0F003C0F003C07003C07003C0700380 3803803807001807000C0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF 8040010080020080020080040000080000080000100000200000200000400000400000C0 0000C00001C0000180000380000380000380000380000780000780000780000780000780 00078000078000030000121F7D9D17>I<03F0000C0C0010060030030020018060018060 01806001807001807803003E03003F06001FC8000FF00003F80007FC000C7E00103F0030 0F806003804001C0C001C0C000C0C000C0C000C0C000806001802001001002000C0C0003 F000121F7E9D17>I<03F0000E18001C0C00380600380700700700700380F00380F00380 F003C0F003C0F003C0F003C0F003C07007C07007C03807C0180BC00E13C003E3C0000380 000380000380000700300700780600780E00700C002018001070000FC000121F7E9D17> I<70F8F8F8700000000000000000000070F8F8F87005147C930D>I<70F8F8F870000000 0000000000000070F0F8F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF0 0000000000000000000000000000000000000000000000000000000000000000FFFFFFF0 7FFFFFE01C0C7D9023>61 D<0FC0307040384038E03CF03CF03C603C0038007000E000C0 01800180010003000200020002000200020002000000000000000000000007000F800F80 0F8007000E207D9F15>63 D<000100000003800000038000000380000007C0000007C000 0007C0000009E0000009E0000009E0000010F0000010F0000010F0000020780000207800 0020780000403C0000403C0000403C0000801E0000801E0000FFFE0001000F0001000F00 01000F00020007800200078002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F 22>65 DI<000FC040007030C001 C009C0038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078 000040F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8 000000780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003 80020001C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E007 0001E00F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8 000000F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C 0001E03C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000 781820000FE0001E217D9F24>I II<0FFF C0007C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C 00F83C00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007 001E0007803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001 F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003 E03C0003C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80 001C217D9F23>II<001F800000 F0F00001C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C 0003E0780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F8 0001F0F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F 204F0007A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000 001FE000001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E000 80E00080E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007 800003C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C 0081F80012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F003080 0F0010800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F000000 0F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F000000 0F0000000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7F F83FF80FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C100000 3C2000003E4000001E4000000F8000000F8000000780000003C0000007E0000005E00000 09F0000018F8000010780000207C0000603C0000401E0000801F0001800F000100078002 0007C0070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80 800F00001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004 0780040F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C> II<080410082010201040204020804080408040B8 5CFC7EFC7E7C3E381C0F0E7B9F17>II<08102020 4040808080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00 000E00000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10 3867200F83C014147E9317>I<0E0000FE00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00 780E00780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E 0015207F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0007000 7000380138011C020E0C03F010147E9314>I<000380003F800003800003800003800003 8000038000038000038000038000038000038003E380061B801C07803803803803807003 80700380F00380F00380F00380F00380F00380F003807003807003803803803807801C07 800E1B8003E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F0 0380F00380FFFF80F00000F00000F000007000007000003800801800800C010007060001 F80011147F9314>I<007C00C6018F038F07060700070007000700070007000700FFF007 00070007000700070007000700070007000700070007000700070007000700070007007F F01020809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F0078 0F00380E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC030 01E0600070C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I< 0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E0000 0E3E000E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C001E003E 001E001C000000000000000000000000000E007E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001F001F0 00E0000000000000000000000000007007F000F000700070007000700070007000700070 00700070007000700070007000700070007000700070007000706070F060F0C061803F00 0C28829E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70000EF8 000F38000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF815207F9F18 >I<0E00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B20809F0C> I<0E1F01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00E00E00 0E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E 00E00E000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I< 0E3E00FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E 001C03803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000 E07000E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F01C00F 00E00E00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E00F00E 00E00F01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E00000E 0000FFE000151D7F9319>I<03E0800619801C05803C0780380380780380700380F00380 F00380F00380F00380F00380F003807003807803803803803807801C0B800E138003E380 000380000380000380000380000380000380000380000380003FF8151D7E9318>I<0E78 FE8C0F1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E00 FFE00F147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070803880 188018C018C018E030D0608F800D147E9312>I<020002000200060006000E000E003E00 FFF80E000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610 031001E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16 147F9319>I II<7FC3FC0F01E00701C007018003810001C20000E4 0000EC00007800003800003C00007C00004E000087000107000303800201C00601E01E01 E0FF07FE1714809318>II<3FFF38 0E200E201C40384078407000E001E001C00380078007010E011E011C0338027006700EFF FE10147F9314>III<30307878F87C 787830300E057C9E17>127 D E /FD 46 122 df<70F8FCFC7404040404080810102040 060F7C840E>44 DI<70F8F8F87005057C840E>I<01F000071C00 0C06001803003803803803807001C07001C07001C07001C0F001E0F001E0F001E0F001E0 F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001C07001C0 7001C07803C03803803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038003800380038003800380038007C0FFFE0F217CA018 >I<03F0000C1C001007002007804003C04003C08003E0F003E0F801E0F801E0F801E020 03E00003E00003C00003C0000780000700000E00001C0000180000300000600000C00001 80000100000200200400200800201800603000403FFFC07FFFC0FFFFC013217EA018>I< 03F8000C1E001007002007804007C07807C07803C07807C03807C0000780000780000700 000F00000E0000380003F000001C00000F000007800007800003C00003C00003E02003E0 7003E0F803E0F803E0F003C04003C0400780200780100F000C1C0003F00013227EA018> I<000200000600000E00000E00001E00001E00002E00004E00004E00008E00008E00010E 00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E00800E00FFFF F8000E00000E00000E00000E00000E00000E00000E00001F0001FFF015217FA018>I<10 00801E07001FFF001FFE001FF80013E00010000010000010000010000010000010000010 F800130E001407001803801003800001C00001C00001E00001E00001E00001E07001E0F0 01E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013227EA018>I< 007E0001C1000300800601C00E03C01C03C0180180380000380000780000700000700000 F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0F001E0F001E0 7001E07001E07001E03801C03801C01803801C03000C0600070C0001F00013227EA018> I<01F800060E000803001001802001802000C06000C06000C06000C07000C07801803E01 003F02001FC4000FF80003F80003FC00067F00083F80100F803007C06001C06000E0C000 E0C00060C00060C00060C000606000406000C03000801803000E0E0003F00013227EA018 >56 D<01F000060C000C0600180700380380700380700380F001C0F001C0F001C0F001E0 F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E003E1E00001C0 0001C00001C0000380000380300300780700780600700C002018001030000FC00013227E A018>I<0001800000018000000180000003C0000003C0000003C0000005E0000005E000 000DF0000008F0000008F0000010F800001078000010780000203C0000203C0000203C00 00401E0000401E0000401E0000800F0000800F0000FFFF000100078001000780030007C0 020003C0020003C0040003E0040001E0040001E00C0000F00C0000F03E0001F8FF800FFF 20237EA225>65 D<0007E0100038183000E0063001C00170038000F0070000F00E000070 1E0000701C0000303C0000303C0000307C0000107800001078000010F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F800000078000000780000107C000010 3C0000103C0000101C0000201E0000200E000040070000400380008001C0010000E00200 00381C000007E0001C247DA223>67 D69 DI73 D77 DI80 D82 D<03F0200C0C601802603001E07000E06000 60E00060E00060E00020E00020E00020F00000F000007800007F00003FF0001FFE000FFF 0003FF80003FC00007E00001E00000F00000F0000070800070800070800070800070C000 60C00060E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF8780780786007 8018400780084007800840078008C007800C800780048007800480078004800780040007 800000078000000780000007800000078000000780000007800000078000000780000007 800000078000000780000007800000078000000780000007800000078000000780000007 80000007800000078000000FC00003FFFF001E227EA123>I<0FE0001838003C0C003C0E 0018070000070000070000070000FF0007C7001E07003C0700780700700700F00708F007 08F00708F00F087817083C23900FC1E015157E9418>97 D<0E0000FE00001E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F000E61C0 0E80600F00300E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E 0E001C0E003C0E00380F00700C80600C41C0083F0017237FA21B>I<01FE000703000C07 801C0780380300780000700000F00000F00000F00000F00000F00000F00000F000007000 007800403800401C00800C010007060001F80012157E9416>I<0000E0000FE00001E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E001F8E007 04E00C02E01C01E03800E07800E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F0 00E07000E07800E03800E01801E00C02E0070CF001F0FE17237EA21B>I<01FC00070700 0C03801C01C03801C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000F00000 7000007800203800201C00400E008007030000FC0013157F9416>I<003C00C6018F038F 030F070007000700070007000700070007000700FFF80700070007000700070007000700 0700070007000700070007000700070007000700070007807FF8102380A20F>I<000070 01F198071E180E0E181C07001C07003C07803C07803C07803C07801C07001C07000E0E00 0F1C0019F0001000001000001800001800001FFE000FFFC00FFFE03800F0600030400018 C00018C00018C000186000306000303800E00E038003FE0015217F9518>I<0E0000FE00 001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E1F800E60C00E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C 001E003E001E001C00000000000000000000000000000000000E00FE001E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A227FA10E >I<01C003E003E003E001C00000000000000000000000000000000001E00FE001E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F18061803E000B2C82A10F>I<0E0000FE00001E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E03FC0E01 F00E01C00E01800E02000E04000E08000E10000E38000EF8000F1C000E1E000E0E000E07 000E07800E03C00E01C00E01E00E00F00E00F8FFE3FE17237FA21A>I<0E00FE001E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B237FA20E>I<0E1F C07F00FE60E183801E807201C00F003C00E00F003C00E00E003800E00E003800E00E0038 00E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800 E00E003800E00E003800E00E003800E00E003800E00E003800E0FFE3FF8FFE27157F942A >I<0E1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F941B>I< 01FC000707000C01801800C03800E0700070700070F00078F00078F00078F00078F00078 F00078F000787000707800F03800E01C01C00E038007070001FC0015157F9418>I<0E1F 00FE61C00E80600F00700E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E00 1E0E001E0E003C0E003C0E00380F00700E80E00E41C00E3F000E00000E00000E00000E00 000E00000E00000E00000E00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F06 0F000E000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF010157F94 13>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C80 0CC00CC008E018D0308FC00E157E9413>I<02000200020002000600060006000E001E00 3E00FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E04 0E040708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E0070 0E00700E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F0060170 03827800FC7F18157F941B>III121 D E /FE 20 118 df45 D68 D73 D77 D80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E0780000E0 700000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F8000000 7C0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC0000FFFF00 000FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F8000000F8 8000007880000078800000788000007880000078C0000078C0000070E00000F0E00000E0 F00000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E003E001E 0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E001F001E 003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E0478003E 047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700 040F00041E001E1C003E3C003E7C003E7C0008780000F80000F80000F80000F80000F800 00F80000F80000F80000F800007800007C00007C00003C00011E00011E00020F00020700 0403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E00000 01E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E00000 01E0000001E0000001E0000001E0000001E0000001E0001F81E000F061E001C019E00780 05E00F0003E00E0003E01E0001E03C0001E03C0001E07C0001E0780001E0F80001E0F800 01E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0780001E07800 01E03C0001E03C0001E01C0001E01E0003E00E0005E0070009E0038011F000E061FF003F 81FF20327DB125>I<003F800000E0E0000380380007003C000E001E001E001E001C000F 003C000F007C000F0078000F8078000780F8000780F8000780FFFFFF80F8000000F80000 00F8000000F8000000F8000000F8000000780000007C0000003C0000003C0000801E0000 800E0001000F0002000780020001C00C0000F03000001FC000191F7E9E1D>I<0007E000 1C1000383800707C00E07C01E07C01C03803C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C000FFFFC0FFFFC003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00007E0007FFF007F FF0016327FB114>I<000000F0007F030801C1C41C0380E81C070070080F0078001E003C 001E003C003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E003C 000F007800070070000780E00009C1C000087F0000180000001800000018000000180000 00180000001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E0300000 F070000070E0000038E0000038E0000038E0000038E00000387000007070000070380000 E01C0001C00700070001C01C00003FE0001E2F7E9F21>I<07000F801F801F800F800700 000000000000000000000000000000000000000000000780FF80FF800F80078007800780 078007800780078007800780078007800780078007800780078007800780078007800780 0780078007800FC0FFF8FFF80D307EAF12>105 D<0780FE001FC000FF83078060F000FF 8C03C18078000F9001E2003C0007A001E4003C0007A000F4001E0007C000F8001E0007C0 00F8001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000 F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0 001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F000 1E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E 000FC001F8003F00FFFC1FFF83FFF0FFFC1FFF83FFF0341F7E9E38>109 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C000F000 07C000F000078000F000078000F000078000F000078000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F0000FC001 F800FFFC1FFF80FFFC1FFF80211F7E9E25>I<001FC00000F0780001C01C00070007000F 0007801E0003C01C0001C03C0001E03C0001E0780000F0780000F0780000F0F80000F8F8 0000F8F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8780000F07C0001F03C 0001E03C0001E01E0003C01E0003C00F00078007800F0001C01C0000F07800001FC0001D 1F7E9E21>I<0783E0FF8C18FF907C0F907C07A07C07C03807C00007C00007C000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0780000780000780000780000780000780000FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E00010E00010F00010F800007E00 003FF0001FFF000FFFC003FFE0003FF00001F80000F880003C80003C80001CC0001CC000 1CE0001CE00018F00038F00030CC0060C301C080FE00161F7E9E1A>I<00400000400000 400000400000400000C00000C00000C00001C00001C00003C00007C0000FC0001FFFE0FF FFE003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C01003C01003C01003C01003C01003C01003C01003 C01001C02001E02000E0400078C0001F00142C7FAB19>I<078000F000FF801FF000FF80 1FF0000F8001F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078001F000 078001F000078001F000038002F00003C004F00001C008F800007030FF80001FC0FF8021 1F7E9E25>I E /FF 5 85 df<00000000C00000000000E00000000001E00000000003E0 0000000003E00000000007E00000000007E0000000000FE0000000000FE0000000001FE0 000000001FE00000000037E00000000067E00000000067E000000000C7E000000000C7F0 0000000183F00000000183F00000000303F00000000703F00000000603F00000000C03F0 0000000C03F00000001803F00000001803F00000003003F00000003003F00000006003F0 000000C003F0000000C003F00000018003F00000018003F8000003FFFFF8000003FFFFF8 0000060001F800000E0001F800000C0001F80000180001F80000180001F80000300001F8 0000300001F80000600001F80000E00001F80000C00001F80001C00001F80001C00001F8 0007C00001FC001FC00003FC00FFF8007FFFE0FFF8007FFFE02B327BB135>65 D<000FFFFFFE0000000FFFFFFF800000007F000FE00000007E0003F00000007E0000F800 00007E0000FC0000007E00007C000000FC00003E000000FC00003E000000FC00003F0000 00FC00001F000001F800001F000001F800001F800001F800001F800001F800001F800003 F000001F800003F000001F800003F000001F800003F000001F800007E000003F800007E0 00003F800007E000003F800007E000003F80000FC000003F00000FC000007F00000FC000 007F00000FC000007F00001F8000007E00001F800000FE00001F800000FE00001F800000 FC00003F000001FC00003F000001F800003F000001F800003F000003F000007E000003E0 00007E000007E000007E00000FC000007E00000F800000FC00001F800000FC00003F0000 00FC00007E000000FC0000FC000001F80001F0000001F80003E0000001F8000FC0000003 F8007F000000FFFFFFFC000000FFFFFFE000000031317BB036>68 D<000FFFFFFFFC000FFFFFFFFC00007F0001FC00007E00007C00007E00003C00007E0000 3C00007E0000180000FC0000180000FC0000180000FC0000180000FC0000180001F80000 180001F80000180001F80000180001F80000180003F00080100003F00180000003F00180 000003F00180000007E00300000007E00300000007E00700000007E01F0000000FFFFE00 00000FFFFE0000000FC01E0000000FC00E0000001F800C0000001F800C0000001F800C00 00001F800C0000003F00180000003F00080000003F00000000003F00000000007E000000 00007E00000000007E00000000007E0000000000FC0000000000FC0000000000FC000000 0000FC0000000001F80000000001F80000000001F80000000003F800000000FFFFF00000 00FFFFF00000002E317BB02F>70 D<000FFFFFF000000FFFFFFE0000007F003F8000007E 000FC000007E0007E000007E0003F000007E0001F80000FC0001F80000FC0001F80000FC 0001F80000FC0001F80001F80003F80001F80003F80001F80003F80001F80003F00003F0 0007F00003F00007E00003F0000FC00003F0000FC00007E0001F000007E0007E000007E0 00FC000007E007F000000FFFFFC000000FFFFF0000000FC00F8000000FC003C000001F80 03E000001F8001F000001F8001F000001F8001F800003F0001F800003F0001F800003F00 01F800003F0001F800007E0003F800007E0003F800007E0003F000007E0003F00000FC00 07F00000FC0007F00000FC0007F00800FC0007F00C01F80007F01801F80007F01801F800 03F03003F80003F030FFFFE001F0E0FFFFE000FFC0000000003F002E327BB034>82 D<07FFFFFFFFF00FFFFFFFFFF00FC00FE003F01E000FC000F01C000FC000E018000FC000 E038000FC0006030001F8000E030001F8000E060001F8000C060001F8000C060003F0000 C0C0003F0000C0C0003F0000C0C0003F0000C080007E00008000007E00000000007E0000 0000007E0000000000FC0000000000FC0000000000FC0000000000FC0000000001F80000 000001F80000000001F80000000001F80000000003F00000000003F00000000003F00000 000003F00000000007E00000000007E00000000007E00000000007E0000000000FC00000 00000FC0000000000FC0000000000FC0000000001F80000000001F80000000001F800000 00001F80000000003F00000000003F00000000003F0000000000FF00000000FFFFFF0000 00FFFFFF0000002C3173B033>84 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 795 908 a FF(D)26 b(R)g(A)f(F)h(T)225 999 y FE(Do)r(cumen)n(t) 20 b(for)i(a)f(Standard)g(Message-P)n(assing)f(In)n(terface)621 1194 y FD(Message)c(P)o(assing)h(In)o(terface)e(F)l(orum)784 1320 y(No)o(v)o(em)o(b)q(er)e(2,)j(1993)77 1378 y(This)h(w)o(ork)f(w)o (as)h(supp)q(orted)g(in)f(part)h(b)o(y)e(ARP)l(A)h(and)h(NSF)e(under)h (gran)o(t)h(ASC-9310330,)i(the)192 1436 y(National)d(Science)f(F)l (oundation)i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q (erativ)o(e)76 1494 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o (y)e(the)h(Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1552 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 166 45 a FC(The)14 b(Message)e(P)o(assing)h(In)o(terface)h(F)l (orum)f(\(MPIF\),)f(with)i(participation)g(from)f(o)o(v)o(er)f(40)h (organi-)75 102 y(zations,)h(has)f(b)q(een)i(meeting)f(since)g(Jan)o (uary)g(1993)e(to)h(discuss)h(and)g(de\014ne)h(a)e(set)g(of)g(library)i (in)o(terface)75 158 y(standards)h(for)h(message)f(passing.)25 b(MPIF)16 b(is)i(not)e(sanctioned)i(or)e(supp)q(orted)h(b)o(y)g(an)o(y) g(o\016cial)g(stan-)75 214 y(dards)e(organization.)166 271 y(This)22 b(is)f(a)g(draft)f(of)h(what)g(will)h(b)q(ecome)g(the)f (Final)h(Rep)q(ort,)h(V)l(ersion)f(1.0,)f(of)f(the)i(Message)75 327 y(P)o(assing)c(In)o(terface)h(F)l(orum.)29 b(This)19 b(do)q(cumen)o(t)g(con)o(tains)g(all)g(the)g(tec)o(hnical)h(features)e (prop)q(osed)h(for)75 384 y(the)c(in)o(terface.)20 b(This)c(cop)o(y)f (of)g(the)g(draft)g(w)o(as)f(pro)q(cessed)i(b)o(y)f(L)1174 378 y FB(a)1195 384 y FC(T)1220 398 y(E)1246 384 y(X)g(on)g(No)o(v)o (em)o(b)q(er)g(2,)g(1993.)166 440 y(MPIF)d(in)o(vites)h(commen)o(ts)e (on)h(the)g(tec)o(hnical)i(con)o(ten)o(t)d(of)h(MPI,)f(as)h(w)o(ell)h (as)e(on)h(the)g(editorial)i(pre-)75 497 y(sen)o(tation)h(in)h(the)f (do)q(cumen)o(t.)20 b(Commen)o(ts)14 b(receiv)o(ed)j(b)q(efore)e(Jan)o (uary)g(15,)f(1994)g(will)j(b)q(e)f(considered)75 553 y(in)g(pro)q(ducing)h(the)e(\014nal)h(draft)e(of)h(V)l(ersion)h(1.0)e (of)h(the)g(Message)g(P)o(assing)g(In)o(terface)g(Sp)q(eci\014cation.) 166 610 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o(terface,) g(simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g(used)75 666 y(standard)g(for)f(writing)i(message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface)g(should)h(establishing)75 723 y(a)d(practical,)h(p)q(ortable,)f(e\016cien)o(t,)g(and)h (\015exible)h(standard)e(for)f(message)h(passing.)179 1978 y(c)166 1979 y FA(\015)p FC(1993)g(Univ)o(ersit)o(y)h(of)f(T)l (ennessee,)h(Kno)o(xville,)h(T)l(ennessee.)22 b(P)o(ermission)16 b(to)f(cop)o(y)g(without)h(fee)75 2035 y(all)e(or)f(part)f(of)h(this)h (material)f(is)h(gran)o(ted,)e(pro)o(vided)i(the)g(Univ)o(ersit)o(y)f (of)g(T)l(ennessee)h(cop)o(yrigh)o(t)f(notice)75 2092 y(and)i(the)h(title)g(of)f(this)g(do)q(cumen)o(t)h(app)q(ear,)f(and)g (notice)h(is)g(giv)o(en)g(that)e(cop)o(ying)i(is)g(b)o(y)f(p)q (ermission)h(of)75 2148 y(the)f(Univ)o(ersit)o(y)h(of)f(T)l(ennessee.) -32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 3 2 2 bop 75 377 a Fy(Con)m(ten)m(ts)75 645 y Fx(Ac)o(kno)o(wledgmen)o (ts)1331 b(vii)75 747 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(1)143 804 y FC(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)91 b FC(1)143 860 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h (Standard?)32 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(2)143 917 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h (Implemen)o(tation?)38 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)91 b FC(3)143 973 y(1.4)46 b(What)14 b(Is)i(Included)h (In)f(The)g(Standard?)42 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(3)143 1030 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f (Standard?)23 b Fw(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(3)143 1086 y(1.6)46 b(Organization)16 b(of)e(this)i(Do)q(cumen)o(t)34 b Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(4)75 1188 y Fx(2)42 b(MPI)17 b(T)l(erms)f(and)i(Con)o(v)o(en)o(tions)1042 b(6)143 1245 y FC(2.1)46 b(Pro)q(cedure)16 b(Sp)q(eci\014cation)44 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(6)143 1302 y(2.2)46 b(Seman)o(tic)15 b(T)l(erms)g Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 91 b FC(6)143 1358 y(2.3)46 b(Data)14 b(T)o(yp)q(es)30 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b FC(7)248 1415 y(2.3.1)50 b(Opaque)16 b(ob)s(jects)21 b Fw(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(7)248 1471 y(2.3.2)50 b(Arra)o(y)14 b(argumen)o(ts)28 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(8)248 1528 y(2.3.3)50 b(State)14 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(8)248 1584 y(2.3.4)50 b(Named)15 b(constan)o(ts)23 b Fw(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(8)248 1641 y(2.3.5)50 b(Choice)20 b Fw(:)i(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(8)248 1697 y(2.3.6)50 b(Addresses)28 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(9)143 1754 y(2.4)46 b(Language)15 b(Binding)45 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)91 b FC(9)248 1810 y(2.4.1)50 b(Data)14 b(T)o(yp)q(es)i(and)f(Naming)g(Con)o(v)o(en)o(tions)39 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)91 b FC(9)248 1867 y(2.4.2)50 b(F)l(ortran)14 b(77)h(Binding)i (Issues)30 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b FC(9)248 1923 y(2.4.3)50 b(C)15 b(Binding)i(Issues)31 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(10)143 1980 y(2.5)46 b(Pro)q(cesses)40 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(11)143 2036 y(2.6)46 b(Error)14 b(Handling)32 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(11)143 2093 y(2.7)46 b(Implemen)o(tation)16 b(issues)36 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(12)248 2149 y(2.7.1)50 b(Indep)q(endence)19 b(of)14 b(basic)i(run)o(time)g (routines)44 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b FC(12)248 2206 y(2.7.2)50 b(In)o(teraction)16 b(with)f(signals)h(in)g(POSIX)42 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(13)75 2308 y Fx(3)42 b(P)o(oin)o(t)17 b(to)h(P)o(oin)o(t)g(Comm)o(unication) 975 b(14)143 2364 y FC(3.1)46 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FC(14)143 2421 y(3.2)46 b(Basic)16 b(send)f(op)q(eration) 27 b Fw(:)c(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(14)248 2477 y(3.2.1)50 b(Message)15 b(data)26 b Fw(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(15)248 2534 y(3.2.2)50 b(Message)15 b(en)o(v)o(elop)q(e)j Fw(:)k(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(16)143 2591 y(3.3)46 b(Basic)16 b(receiv)o(e)g(op)q(eration)h Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(17)248 2647 y(3.3.1)50 b(Return)16 b(status)j Fw(:)k(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(18)143 2704 y(3.4)46 b(Seman)o(tics)15 b(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)40 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FC(19)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 4 3 3 bop 143 45 a FC(3.5)46 b(Data)14 b(T)o(yp)q(e)h(Matc)o(hing)22 b Fw(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(21)143 102 y(3.6)46 b(Data)14 b(con)o(v)o(ersion)g Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FC(23)143 159 y(3.7)46 b(Comm)o(unication)15 b(Mo)q(des)i Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(24)143 216 y(3.8)46 b(Non)o(blo)q(c)o(king)16 b(comm)o(unication)j Fw(:)j(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(27)248 272 y(3.8.1)50 b(Comm)o(unication)16 b(Ob)s(jects)23 b Fw(:)g(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(27)248 329 y(3.8.2)50 b(Comm)o(unication)16 b(initiation)32 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b FC(27)248 386 y(3.8.3)50 b(Comm)o(unication)16 b(Completion)i Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(30)248 443 y(3.8.4)50 b(Seman)o(tics)16 b(of)f(Non)o(blo)q(c)o(king)h(Comm)o(unications)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(31)248 500 y(3.8.5)50 b(Multiple)17 b(Completions)41 b Fw(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(33)143 556 y(3.9)46 b(Prob)q(e)15 b(and)g(Cancel)g Fw(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(36)143 613 y(3.10)23 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(ob)s(jects)i Fw(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(38)143 670 y(3.11)23 b(Send-receiv)o(e)d Fw(:)i(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(42)143 727 y(3.12)23 b(Null)16 b(pro)q(cesses)i Fw(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(44)143 784 y(3.13)23 b(Deriv)o(ed)15 b(datat)o(yp)q(es)42 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(44)248 840 y(3.13.1)27 b(Datat)o(yp)q(e)14 b(constructors)27 b Fw(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b FC(46)248 897 y(3.13.2)27 b(Address)16 b(and)f(exten)o(t)g (functions)32 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(52)248 954 y(3.13.3)27 b(Lo)o(w)o(er-b)q(ound)16 b(and)f(upp)q(er-b)q(ound)j (mark)o(ers)33 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FC(53)248 1011 y(3.13.4)27 b(Commit)15 b(and)g(free)28 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(54)248 1068 y(3.13.5)27 b(Use)16 b(of)e(general)i(datat)o(yp)q(es) f(in)h(comm)o(unication)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)69 b FC(55)248 1124 y(3.13.6)27 b(Examples)33 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(58)248 1181 y(3.13.7)27 b(Correct)15 b(use)g(of)g(addresses)26 b Fw(:)d(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(65)143 1238 y(3.14)23 b(Univ)o(ersal)16 b(comm)o(unication)g(functions)24 b Fw(:)f(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b FC(66)248 1295 y(3.14.1)27 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(ob)s(jects)e Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FC(67)248 1352 y(3.14.2)27 b(Non)o(blo)q(c)o(king) 17 b(comm)o(unication)e(initiation)31 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(68)248 1408 y(3.14.3)27 b(Comm)o(unication)16 b(completion)31 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(68)248 1465 y(3.14.4)27 b(Blo)q(c)o(king)17 b(comm)o(unication)g Fw(:)23 b(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FC(68)248 1522 y(3.14.5)27 b(Prob)q(e)16 b(and)f(cancel)23 b Fw(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(68)248 1579 y(3.14.6)27 b(Return)16 b(status)j Fw(:)k(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(68)248 1636 y(3.14.7)27 b(send-receiv)o(e)17 b(and)e(exc)o(hange)22 b Fw(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(69)248 1692 y(3.14.8)27 b(Deriv)o(ed)16 b(datat)o(yp)q(es)37 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b FC(69)75 1795 y Fx(4)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(70)143 1852 y FC(4.1)46 b(In)o(tro)q(duction) 15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b FC(70)143 1908 y(4.2)46 b(Comm)o(unication)15 b(F)l(unctions)24 b Fw(:)f(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(71)143 1965 y(4.3)46 b(Barrier)15 b(sync)o(hronization)44 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(71)143 2022 y(4.4)46 b(Data)14 b(mo)o(v)o(e)g(functions)29 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(71)248 2079 y(4.4.1)50 b(Broadcast)24 b Fw(:)e(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(73)248 2136 y(4.4.2)50 b(Gather)13 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(73)248 2192 y(4.4.3)50 b(Examples)16 b(of)f(Usage)f(of)h Fv(MPI)p 921 2192 14 2 v 16 w(GA)l(THER)p FC(,)h Fv(MPI)p 1223 2192 V 16 w(GA)l(THERV)24 b Fw(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(75)248 2249 y(4.4.4)50 b(Scatter)c Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(82)248 2306 y(4.4.5)50 b(Examples)16 b(of)f(Usage)f(of)h Fv(MPI)p 921 2306 V 16 w(SCA)l(TTER)p FC(,)h Fv(MPI)p 1246 2306 V 16 w(SCA)l(TTERV)d Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(84)248 2363 y(4.4.6)50 b(Gather-to-all)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b FC(86)248 2420 y(4.4.7)50 b(All-to-All)18 b(Scatter/Gather)32 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(88)143 2476 y(4.5)46 b(Global)16 b(Compute)e(Op)q(erations)20 b Fw(:)i(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(90)248 2533 y(4.5.1)50 b(Reduce)45 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(90)248 2590 y(4.5.2)50 b(User-Reduce)c Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b FC(93)248 2647 y(4.5.3)50 b(All-Reduce)43 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(96)248 2704 y(4.5.4)50 b(Reduce-Scatter)30 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(97)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 5 4 4 bop 248 45 a FC(4.5.5)50 b(Scan)25 b Fw(:)d(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b FC(99)143 102 y(4.6)46 b(Correctness)32 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(100)248 159 y(4.6.1)k(Sync)o(hronization)17 b(Side-E\013ects)12 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(100)248 216 y(4.6.2)k(Multiple)17 b(Calls)f(to)f(Collectiv)o(e)31 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(101)75 319 y Fx(5)c(Groups,)17 b(Con)o(texts,)f(Comm)o(unicators,) g(and)i(Cac)o(hing)543 b(104)143 376 y FC(5.1)46 b(In)o(tro)q(duction) 15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b FC(104)248 433 y(5.1.1)k(Wh)o(y)15 b(w)o(e)g(need)h (libraries)30 b Fw(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(104)248 490 y(5.1.2)k(Wh)o(y)15 b(existing)h(systems)f(do)g(not)g (supp)q(ort)g(libraries)f Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)46 b FC(104)248 547 y(5.1.3)k(What)15 b(features)g(are)f (needed)j(to)d(supp)q(ort)i(libraries)23 b Fw(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(105)248 604 y(5.1.4)k(What)15 b(these)g(features)g(are)g(called)i(in)f(MPI)27 b Fw(:)c(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(105)143 661 y(5.2)g(Basic)16 b(Concepts)30 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(106)248 718 y(5.2.1)k(Groups)43 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(106)248 775 y(5.2.2)k(Con)o(texts)c Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(106)248 832 y(5.2.3)k(Comm)o(unicators) 17 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(107)248 890 y(5.2.4)k(Prede\014ned)17 b(Comm)o(unicators)38 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(107)143 947 y(5.3)g(Group)15 b(Managemen)o(t)41 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FC(107)248 1004 y(5.3.1)k(Group)15 b(Accessors)34 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(107)248 1061 y(5.3.2)k(Group)15 b(Constructors)37 b Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(108)248 1118 y(5.3.3)k(Group)15 b(Destructors)27 b Fw(:)c(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FC(112)143 1175 y(5.4)g(Comm)o(unicator)14 b(Managemen)o(t)i Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(112)248 1232 y(5.4.1)k(Comm)o(unicator)15 b(Accessors)43 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(112)248 1289 y(5.4.2)k(Comm)o (unicator)15 b(Constructors)10 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(113)248 1346 y(5.4.3)k(Comm)o(unicator)15 b(Destructors)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(115)143 1403 y(5.5)g(In)o(ter-Comm)o (unication)14 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FC(115)248 1460 y(5.5.1)k(Comm)o(unicator)15 b(Accessors)43 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(117)248 1517 y(5.5.2)k(In)o(tercomm)o(unicator)15 b(Constructors)f(and)h (Destructors)j Fw(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(117)248 1574 y(5.5.3)k(Name)15 b(Service)25 b Fw(:)e(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(120)143 1631 y(5.6)g(Cac)o(hing)32 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(120)248 1688 y(5.6.1)k(F)l(unctionalit)o(y)32 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(120)143 1745 y(5.7)g(F)l (ormalizing)16 b(the)f(Lo)q(osely)h(Sync)o(hronous)g(Mo)q(del)g (\(Usage,)e(Safet)o(y\))j Fw(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(125)248 1802 y(5.7.1)k(Basic)16 b(Statemen)o(ts)22 b Fw(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(125)248 1859 y(5.7.2)k(Mo)q(dels)16 b(of)f(Execution)29 b Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(125)143 1916 y(5.8)g(Motiv)m(ating)15 b(Examples)h Fw(:)23 b(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(127)248 1973 y(5.8.1)k(Curren)o (t)15 b(Practice)g(#1)27 b Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(127)248 2030 y(5.8.2)k(Curren)o(t)15 b(Practice)g(#2)27 b Fw(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(128)248 2087 y(5.8.3)k(\(Appro)o(ximate\))15 b(Curren)o(t)f(Practice)i(#3)35 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b FC(128)248 2144 y(5.8.4)k(Example)16 b(#4)44 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(129)248 2201 y(5.8.5)k(Library)16 b(Example)g(#1)22 b Fw(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(130)248 2258 y(5.8.6)k(Library)16 b(Example)g(#2)22 b Fw(:)g(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FC(131)248 2316 y(5.8.7)k(In)o(ter-Comm)o (unication)16 b(Examples)g Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(133)75 2418 y Fx(6)c(Pro)q(cess)17 b(T)l(op)q(ologies)1224 b(139)143 2475 y FC(6.1)46 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(139)143 2532 y(6.2)g(Virtual)16 b(T)l(op)q(ologies)37 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(139)143 2590 y(6.3)g(Em)o(b)q(edding)16 b(in)g(MPI)22 b Fw(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(140)143 2647 y(6.4)g(Ov)o(erview)16 b(of)e(the)i(prop)q(osed)f (MPI)g(functions)20 b Fw(:)j(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(140)143 2704 y(6.5)g(T)l(op)q(ology)15 b(Constructors)20 b Fw(:)i(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(141)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 6 5 5 bop 248 45 a FC(6.5.1)50 b(T)l(op)q(ology)15 b(inquiry)i(functions) 42 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(144)248 102 y(6.5.2)k(Cartesian) 15 b(shift)h(co)q(ordinates)43 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(148)248 158 y(6.5.3)k(P)o(artitioning)16 b(of)e(Cartesian)h (structures)45 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b FC(149)248 215 y(6.5.4)k(Lo)o(w{lev)o(el)16 b(top)q(ology)f(functions)37 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(150)143 271 y(6.6)g(A)15 b(simple)i(example)f Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(151)75 373 y Fx(7)c(MPI)17 b(En)o(vironmen)o(tal)g(Managemen)o(t) 883 b(153)143 430 y FC(7.1)46 b(Implemen)o(tation)16 b(information)22 b Fw(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(153)248 486 y(7.1.1)k(En)o(vironmen)o(tal)16 b(Inquiries)28 b Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(153)248 543 y(7.1.2)k(Bu\013ering)42 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b FC(154)143 599 y(7.2)g(Error)14 b(handling)41 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FC(158)143 656 y(7.3)g(Startup)40 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FC(160)75 758 y Fx(8)c(Pro\014ling)18 b(In)o(terface)1238 b(162)143 815 y FC(8.1)46 b(Requiremen)o(ts)28 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FC(162)143 871 y(8.2)g(Discussion)21 b Fw(:)i(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(162)143 928 y(8.3)g(Logic)16 b(of)e(the)i(design)29 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(163)143 984 y(8.4)g(Examples)36 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(163)248 1041 y(8.4.1)k(Pro\014ler)16 b(implemen)o(tation)35 b Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(163)248 1097 y(8.4.2)k(MPI)15 b(library)h(implemen)o(tation)h Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FC(164)143 1154 y(8.5)g(Multiple)17 b(lev)o(els)f(of)f(in)o(terception)25 b Fw(:)d(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b FC(165)75 1256 y Fx(9)c(Initial)20 b(Implemen)o(tation)e (Subset)967 b(166)143 1312 y FC(9.1)46 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b FC(166)143 1369 y(9.2)g(Criteria)15 b(and)h(Rationale)28 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(166)143 1425 y(9.3)g(Subset)15 b(F)l(unctionalit)o(y)31 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(167)248 1482 y(9.3.1)k(P)o(oin)o(t)15 b(to)g(P)o(oin)o(t)f(F)l (unctionalit)o(y)i Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(167)248 1538 y(9.3.2)k(Collectiv)o(e)17 b(Comm)o(unication)e(F)l(unctionalit)o (y)24 b Fw(:)e(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)46 b FC(169)248 1595 y(9.3.3)k(Groups,)15 b(Con)o(texts,)e(Comm)o (unicators,)h(and)i(Cac)o(heing)f Fw(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)46 b FC(169)248 1652 y(9.3.4)k(T)l(op)q(ology)15 b(F)l(unctionalit)o(y)46 b Fw(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(170)248 1708 y(9.3.5)k(Language)15 b(Binding)41 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(171)248 1765 y(9.3.6)k(MPI)15 b(En)o(vironmen)o(tal)h(Managemen)o(t)f Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FC(171)248 1821 y(9.3.7)k(Pro\014ling)16 b(F)l(unctionalit)o(y)24 b Fw(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(171)143 1878 y(9.4)g(Subset)15 b(T)l(esting)42 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b FC(172)75 1980 y Fx(Bibliograph)o(y)1430 b(173)75 2082 y(A)28 b(Language)19 b(Binding)1242 b(175)143 2138 y FC(A.1)35 b(In)o(tro)q(duction)15 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(175)143 2195 y(A.2)35 b(De\014ned)16 b(Constan)o(ts)e(for)g(C)h(and)h(F)l (ortran)21 b Fw(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(175)143 2251 y(A.3)35 b(C)15 b(bindings)i(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o (unication)21 b Fw(:)i(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b FC(177)143 2308 y(A.4)35 b(C)15 b(Bindings)i(for)d (Collectiv)o(e)j(Comm)o(unication)34 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(180)143 2364 y(A.5)35 b(C)15 b(Bindings)i(for)d(Groups,)h(Con)o (texts,)f(and)h(Comm)o(unicators)26 b Fw(:)c(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b FC(181)143 2421 y(A.6)35 b(C)15 b(Bindings)i(for)d(Pro)q(cess)h(T)l(op)q(ologies)43 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(182)143 2477 y(A.7)35 b(C)15 b(bindings)i(for)d(En)o(vironmen)o(tal)i(Inquiry)42 b Fw(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b FC(183)143 2534 y(A.8)35 b(F)l(ortran)14 b(Bindings)j(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o(unication)39 b Fw(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(183)143 2591 y(A.9)35 b(F)l(ortran)14 b(Bindings)j(for)d (Collectiv)o(e)j(Comm)o(unication)23 b Fw(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(186)143 2647 y(A.10)12 b(F)l(ortran)i(Bindings)j(for)d(Groups,)h(Con)o(texts,)f(and) h(Comm)o(unicators)g Fw(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(189)143 2704 y(A.11)12 b(F)l(ortran)i(Bindings)j(for)d(Pro)q(cess) h(T)l(op)q(ologies)32 b Fw(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(190)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 7 6 6 bop 143 45 a FC(A.12)12 b(F)l(ortran)i(Bindings)j(for)d(En)o (vironmen)o(tal)i(Inquiry)24 b Fw(:)e(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b FC(191)75 147 y Fx(MPI)17 b(F)l(unction)h(Index)1252 b(193)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 8 7 7 bop 75 75 a Fy(Ac)m(kno)m(wledgmen)m(ts)166 282 y FC(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f(w)o(as)e(carried)i(out)f (b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o(as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d) h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P)o(assing)g(In)o (terface)75 395 y(\(MPI\),)f(man)o(y)h(p)q(eople)i(serv)o(ed)e(in)h(p)q (ositions)g(of)f(resp)q(onsibilit)o(y:)143 483 y FA(\017)23 b FC(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o(er,)g(Con)o(v)o (eners)g(and)g(Meeting)h(Chairs)143 572 y FA(\017)23 b FC(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes)143 661 y FA(\017)23 b FC(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i (Lusk,)f(P)o(oin)o(t)g(to)g(P)o(oin)o(t)f(Comm)o(unications)143 749 y FA(\017)23 b FC(Al)16 b(Geist,)e(Marc)h(Snir,)h(Stev)o(e)f(Otto,) f(Collectiv)o(e)j(Comm)o(unications)143 838 y FA(\017)23 b FC(Rolf)15 b(Hemp)q(el,)i(Pro)q(cess)e(T)l(op)q(ologies)143 927 y FA(\017)23 b FC(Ewing)15 b(Lusk,)h(Language)f(Binding)143 1015 y FA(\017)23 b FC(William)17 b(Gropp,)d(En)o(vironmen)o(tal)i (Managemen)o(t)e(and)h(Inquiry)143 1104 y FA(\017)23 b FC(James)15 b(Co)o(wnie,)g(Pro\014ling)143 1193 y FA(\017)23 b FC(T)l(on)o(y)16 b(Skjellum,)j(Lyndon)f(Clark)o(e,)f(Marc)g(Snir,)h (Ric)o(hard)g(Little\014eld,)h(Mark)e(Sears,)g(Groups,)189 1249 y(Con)o(texts,)c(and)j(Comm)o(unicators)143 1338 y FA(\017)23 b FC(Stev)o(en)15 b(Huss-Lederman,)h(Initial)h(Implemen)o (tation)f(Subset)143 1427 y FA(\017)23 b FC(Stev)o(e)15 b(Otto,)f(Editor)166 1515 y(The)22 b(follo)o(wing)h(list)f(includes)i (some)e(of)f(the)h(activ)o(e)g(participan)o(ts)h(in)f(the)g(MPI)g(pro)q (cess)g(not)75 1572 y(men)o(tioned)16 b(ab)q(o)o(v)o(e:)100 1705 y(Ed)f(Anderson)134 b(Jo)q(e)16 b(Baron)171 b(Eric)15 b(Barszcz)162 b(Scott)15 b(Berryman)100 1762 y(Rob)g(Bjornson)119 b(Anne)16 b(Elster)138 b(Jim)16 b(F)l(eeney)185 b(Vince)17 b(F)l(ernando)100 1818 y(Sam)e(Fineb)q(erg)118 b(Jon)16 b(Flo)o(w)o(er)154 b(Daniel)16 b(F)l(ry)o(e)179 b(Ian)16 b(Glendinning)100 1875 y(Adam)f(Green)o(b)q(erg)53 b(Rob)q(ert)16 b(Harrison)50 b(Leslie)17 b(Hart)189 b(T)l(om)15 b(Haupt)100 1931 y(Don)g(Heller)179 b(T)l(om)15 b(Henderson)62 b(Alex)16 b(Ho)246 b(C.T.)14 b(Ho)o(w)o(ard)g(Ho)100 1988 y(John)i(Kap)q(enga)106 b(Bob)16 b(Leary)166 b(Arth)o(ur)14 b(Maccab)q(e)84 b(P)o(eter)15 b(Madams)100 2044 y(Alan)h(Main)o(w)o(aring)49 b(Oliv)o(er)17 b(McBry)o(an)55 b(Phil)16 b(McKinley)123 b(Charles)16 b(Mosher)100 2100 y(Dan)f(Nessett)149 b(P)o(eter)15 b(P)o(ac)o(heco)94 b(Ho)o(w)o(ard)14 b(P)o(almer)100 b(P)o(aul)15 b(Pierce)100 2157 y(Sanja)o(y)g(Rank)m(a)120 b(P)o(eter)15 b(Rigsb)q(ee)102 b(Arc)o(h)15 b(Robison)139 b(Eric)o(h)16 b(Sc)o(hikuta)100 2213 y(Am)o(buj)f(Singh)133 b(Alan)16 b(Sussman)93 b(Rob)q(ert)15 b(T)l(omlinson)51 b(Rob)q(ert)16 b(G.)e(V)l(oigt)100 2270 y(Dennis)i(W)l(eeks)117 b(Stephen)17 b(Wheat)67 b(Stev)o(e)15 b(Zenith)166 2412 y(Oak)21 b(Ridge)h(National)g(Lab)q (oratory)e(made)h(the)g(draft)f(a)o(v)m(ailable)j(b)o(y)e(anon)o(ymous) g(FTP)f(mail)75 2469 y(serv)o(ers)15 b(and)g(w)o(as)g(instrumen)o(tal)g (in)h(distributing)h(the)e(do)q(cumen)o(t.)166 2525 y(MPI)22 b(op)q(erated)h(on)f(a)g(v)o(ery)g(tigh)o(t)g(budget)h(\(in)g(realit)o (y)l(,)h(it)f(had)f(no)g(budget)h(when)g(the)f(\014rst)75 2582 y(meeting)14 b(w)o(as)e(announced\).)20 b(D)o(ARP)l(A)13 b(and)g(NSF)g(ha)o(v)o(e)g(supp)q(orted)g(researc)o(h)g(at)f(v)m (arious)i(institutions)75 2638 y(that)i(ha)o(v)o(e)g(made)h(a)f(con)o (tribution)i(to)o(w)o(ards)d(tra)o(v)o(el)h(for)g(the)h(U.S.)f (academics.)25 b(Supp)q(ort)17 b(for)f(sev)o(eral)75 2695 y(Europ)q(ean)g(participan)o(ts)f(w)o(as)g(pro)o(vided)h(b)o(y)f (ESPRIT.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 1 9 1 8 bop 75 362 a Fu(Chapter)34 b(1)75 576 y Fy(In)m(tro)s(duction)41 b(to)g(MPI)75 823 y Ft(1.1)59 b(Overview)19 b(and)g(Goals)75 936 y FC(Message)11 b(passing)i(is)f(a)g(paradigm)f(used)i(widely)g(on) f(certain)h(classes)f(of)f(parallel)j(mac)o(hines;)f(esp)q(ecially)75 992 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1049 y(pro)q(cesses)j(comm)o(unicating)h(through)e(messages)h(is)g (w)o(ell)h(understo)q(o)q(d.)34 b(Ov)o(er)20 b(the)g(last)g(ten)g(y)o (ears,)75 1105 y(substan)o(tial)11 b(progress)f(has)g(b)q(een)h(made)g (in)g(casting)f(signi\014can)o(t)i(applications)g(in)f(this)g (paradigm.)18 b(Eac)o(h)75 1161 y(v)o(endor)13 b(has)g(implemen)o(ted)i (its)e(o)o(wn)g(v)m(arian)o(t.)19 b(More)13 b(recen)o(tly)l(,)h(sev)o (eral)f(systems)g(ha)o(v)o(e)f(demonstrated)75 1218 y(that)k(a)h (message)f(passing)h(system)g(can)g(b)q(e)g(e\016cien)o(tly)h(and)f(p)q (ortably)h(implemen)o(ted.)26 b(It)17 b(is)g(th)o(us)g(an)75 1274 y(appropriate)c(time)g(to)f(try)h(to)f(de\014ne)i(b)q(oth)f(the)g (syn)o(tax)f(and)h(seman)o(tics)g(of)g(a)g(core)f(of)h(library)h (routines)75 1331 y(that)i(will)i(b)q(e)f(useful)h(to)e(a)g(wide)h (range)f(of)h(users)f(and)h(e\016cien)o(tly)h(implemen)o(table)g(on)f (a)f(wide)h(range)75 1387 y(of)e(computers.)166 1450 y(In)21 b(designing)i(MPI)e(w)o(e)f(ha)o(v)o(e)h(sough)o(t)f(to)g(mak)o (e)g(use)h(of)g(the)g(most)f(attractiv)o(e)f(features)i(of)f(a)75 1506 y(n)o(um)o(b)q(er)12 b(of)g(existing)h(message)e(passing)h (systems,)g(rather)f(than)h(selecting)h(one)f(of)g(them)g(and)g (adopting)75 1563 y(it)f(as)f(the)h(standard.)18 b(Th)o(us,)11 b(MPI)g(has)f(b)q(een)i(strongly)f(in\015uenced)i(b)o(y)d(w)o(ork)g(at) g(the)h(IBM)g(T.)f(J.)h(W)l(atson)75 1619 y(Researc)o(h)18 b(Cen)o(ter)g([1)o(,)g(2)o(],)g(In)o(tel's)g(NX/2)g([18)o(],)f(Express) h([17)o(],)g(nCUBE's)g(V)l(ertex)g([15)o(],)f(P4)h([5)o(],)g(and)75 1675 y(P)l(ARMA)o(CS)d([4,)f(6].)19 b(Other)c(imp)q(ortan)o(t)g(con)o (tributions)h(ha)o(v)o(e)e(come)h(from)f(Zip)q(co)q(de)j([19)o(,)d (20],)g(Chimp)75 1732 y([11)o(,)h(12)o(],)g(PVM)g([3)o(,)g(9)o(],)g (and)g(PICL)h([14)o(].)166 1794 y(The)d(MPI)g(standardization)g (e\013ort)f(in)o(v)o(olv)o(ed)i(ab)q(out)f(60)f(p)q(eople)i(from)f(40)f (organizations)h(mainly)75 1851 y(from)k(the)g(United)i(States)e(and)h (Europ)q(e.)27 b(Most)16 b(of)h(the)h(ma)s(jor)e(v)o(endors)h(of)g (concurren)o(t)h(computers)75 1907 y(w)o(ere)10 b(in)o(v)o(olv)o(ed)i (in)f(MPI,)g(along)f(with)i(researc)o(hers)e(from)g(univ)o(ersities,)j (go)o(v)o(ernmen)o(t)d(lab)q(oratories,)h(and)75 1964 y(industry)l(.)21 b(The)14 b(standardization)h(pro)q(cess)f(b)q(egan)h (with)f(the)h(W)l(orkshop)f(on)g(Standards)g(for)g(Message)75 2020 y(P)o(assing)j(in)i(a)e(Distributed)h(Memory)f(En)o(vironmen)o(t,) h(sp)q(onsored)g(b)o(y)f(the)h(Cen)o(ter)f(for)g(Researc)o(h)h(on)75 2077 y(P)o(arallel)11 b(Computing,)g(held)h(April)f(29-30,)f(1992,)g (in)h(Williamsburg,)h(Virginia)g([21)o(].)18 b(A)o(t)9 b(this)i(w)o(orkshop)75 2133 y(the)17 b(basic)h(features)f(essen)o (tial)h(to)f(a)g(standard)g(message)g(passing)g(in)o(terface)h(w)o(ere) f(discussed,)i(and)e(a)75 2190 y(w)o(orking)e(group)g(established)i(to) d(con)o(tin)o(ue)i(the)f(standardization)h(pro)q(cess.)166 2252 y(A)e(preliminary)i(draft)e(prop)q(osal,)g(kno)o(wn)g(as)g(MPI1,)g (w)o(as)f(put)h(forw)o(ard)f(b)o(y)h(Dongarra,)f(Hemp)q(el,)75 2308 y(Hey)l(,)i(and)g(W)l(alk)o(er)f(in)i(No)o(v)o(em)o(b)q(er)e (1992,)f(and)i(a)f(revised)i(v)o(ersion)f(w)o(as)f(completed)h(in)h(F)l (ebruary)e(1993)75 2365 y([10)o(].)19 b(MPI1)c(em)o(b)q(o)q(died)h(the) f(main)h(features)e(that)g(w)o(ere)h(iden)o(ti\014ed)i(at)d(the)h (Williamsburg)i(w)o(orkshop)75 2421 y(as)g(b)q(eing)h(necessary)g(in)g (a)f(message)f(passing)i(standard.)26 b(This)17 b(prop)q(osal)h(w)o(as) e(in)o(tended)j(to)e(initiate)75 2478 y(discussion)i(of)e (standardization)h(issues)h(within)f(the)g(distributed)h(memory)e (concurren)o(t)g(computing)75 2534 y(comm)o(unit)o(y)l(.)30 b(Since)20 b(MPI1)e(w)o(as)g(primarily)i(in)o(tended)g(to)e(promote)f (discussion)k(and)d(\\get)g(the)h(ball)75 2591 y(rolling,")h(it)f(fo)q (cused)g(mainly)g(on)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unications.) 31 b(MPI1)18 b(did)h(not)f(include)j(an)o(y)75 2647 y(collectiv)o(e)e (comm)o(unication)e(routines.)26 b(MPI1)16 b(brough)o(t)g(to)h(the)g (forefron)o(t)e(a)i(n)o(um)o(b)q(er)g(of)f(imp)q(ortan)o(t)75 2704 y(standardization)e(issues;)h(ho)o(w)o(ev)o(er,)d(its)i(ma)s(jor)e (de\014ciency)k(w)o(as)d(that)f(the)i(managemen)o(t)f(of)g(resources) -32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 10 2 9 bop 75 -100 a FC(2)903 b Fs(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)75 45 y FC(is)h(not)f(thread-safe.)166 102 y(In)j(No)o(v)o(em)o(b)q(er)e(1992,)g(a)h(meeting)h(of)e(the)h(MPI)g(w) o(orking)g(group)g(w)o(as)f(held)i(in)g(Minneap)q(olis,)i(at)75 158 y(whic)o(h)15 b(it)g(w)o(as)e(decided)k(to)c(place)j(the)e (standardization)h(pro)q(cess)g(on)f(a)g(more)g(formal)g(fo)q(oting,)g (and)h(to)75 214 y(generally)i(adopt)f(the)g(pro)q(cedures)i(and)e (organization)g(of)g(the)g(High)h(P)o(erformance)f(F)l(ortran)f(F)l (orum.)75 271 y(Sub)q(committees)i(w)o(ere)e(formed)g(for)g(the)h(ma)s (jor)e(comp)q(onen)o(t)i(areas)f(of)g(the)h(standard,)f(and)g(an)h (email)75 327 y(discussion)23 b(service)f(established)g(for)f(eac)o(h.) 37 b(In)22 b(addition,)h(the)e(goal)g(of)g(pro)q(ducing)h(a)f(draft)f (MPI)75 384 y(standard)f(b)o(y)h(the)f(F)l(all)h(of)g(1993)e(w)o(as)h (set.)32 b(T)l(o)19 b(ac)o(hiev)o(e)i(this)f(goal)f(the)g(MPI)h(w)o (orking)f(group)g(met)75 440 y(ev)o(ery)d(6)f(w)o(eeks)g(for)g(t)o(w)o (o)g(da)o(ys)g(throughout)g(the)h(\014rst)f(9)g(mon)o(ths)g(of)h(1993,) e(and)i(presen)o(ted)g(the)g(draft)75 497 y(MPI)j(standard)g(at)g(the)g (Sup)q(ercomputing)i(93)e(conference)h(in)g(No)o(v)o(em)o(b)q(er)f (1993.)31 b(These)20 b(meetings)75 553 y(and)f(the)f(email)h (discussion)i(together)c(constituted)i(the)g(MPI)f(F)l(orum,)g(mem)o(b) q(ership)i(of)e(whic)o(h)h(has)75 610 y(b)q(een)d(op)q(en)g(to)f(all)h (mem)o(b)q(ers)f(of)g(the)g(high)h(p)q(erformance)g(computing)f(comm)o (unit)o(y)l(.)166 666 y(The)j(main)h(adv)m(an)o(tages)e(of)h (establishing)i(a)e(message)g(passing)g(standard)g(are)g(p)q(ortabilit) o(y)h(and)75 723 y(ease-of-use.)h(In)14 b(a)g(distributed)h(memory)e (comm)o(unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f(higher)h (lev)o(el)75 779 y(routines)c(and/or)g(abstractions)f(are)h(build)i(up) q(on)e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i(routines)f(the)g(b) q(ene\014ts)75 835 y(of)22 b(standardization)h(are)f(particularly)h (apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g (message)75 892 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop)q (osed)g(here,)h(pro)o(vides)f(v)o(endors)h(with)f(a)g(clearly)h (de\014ned)h(base)75 948 y(set)10 b(of)g(routines)g(that)g(they)g(can)h (implemen)o(t)g(e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o (vide)f(hardw)o(are)g(supp)q(ort)75 1005 y(for,)k(thereb)o(y)h (enhancing)i(scalabilit)o(y)l(.)166 1061 y(The)h(goal)g(of)g(the)g (Message)g(P)o(assing)g(In)o(terface)g(simply)i(stated)d(is)i(to)f(dev) o(elop)h(a)f(widely)h(used)75 1118 y(standard)e(for)f(writing)i (message-passing)f(programs.)24 b(As)17 b(suc)o(h)h(the)f(in)o(terface) g(should)h(establishing)75 1174 y(a)d(practical,)h(p)q(ortable,)f (e\016cien)o(t,)g(and)h(\015exible)h(standard)e(for)f(message)h (passing.)166 1231 y(A)g(complete)h(list)g(of)f(goals)g(follo)o(ws.)143 1330 y FA(\017)23 b FC(Design)11 b(an)f(application)i(programming)e(in) o(terface)h(\(not)f(necessarily)i(for)d(compilers)j(or)e(a)g(system)189 1386 y(implemen)o(tation)16 b(library\).)143 1478 y FA(\017)23 b FC(Allo)o(w)15 b(e\016cien)o(t)h(comm)o(unication:)21 b(Av)o(oid)16 b(memory)e(to)h(memory)g(cop)o(ying)g(and)h(allo)o(w)f(o) o(v)o(erlap)189 1535 y(of)c(computation)h(and)g(comm)o(unication)h(and) f(o\017oad)f(to)g(comm)o(unication)i(co-pro)q(cessor,)f(where)189 1591 y(a)o(v)m(ailable.)143 1683 y FA(\017)23 b FC(Allo)o(w)15 b(for)g(implemen)o(tation)h(that)f(can)g(b)q(e)h(used)g(in)g(a)f (heterogeneous)g(en)o(vironmen)o(t.)143 1775 y FA(\017)23 b FC(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C)f(and)h(F)l(ortran)e(77)g (bindings)j(for)e(the)g(in)o(terface.)143 1867 y FA(\017)23 b FC(Assume)15 b(a)g(reliable)i(comm)o(unication)f(in)o(terface:)k (User)15 b(need)h(not)f(cop)q(e)h(with)f(comm)o(unication)189 1923 y(failures.)21 b(Suc)o(h)15 b(failures)i(are)d(dealt)i(with)g(b)o (y)f(the)g(underlying)i(comm)o(unication)f(subsystem.)143 2015 y FA(\017)23 b FC(De\014ne)f(an)g(in)o(terface)g(that)f(is)h(not)f (to)q(o)g(di\013eren)o(t)h(from)f(curren)o(t)h(practice,)h(suc)o(h)f (as)g(PVM,)189 2071 y(Express,)14 b(P4,)h(etc)g(and)h(pro)o(vides)f (extension)h(that)f(allo)o(w)g(greater)f(\015exibili)q(t)o(y)l(.)143 2163 y FA(\017)23 b FC(De\014ne)d(an)f(in)o(terface)h(that)e(can)i(b)q (e)g(implemen)o(ted)h(on)e(man)o(y)g(v)o(endor's)g(platforms,)g(with)h (no)189 2220 y(signi\014can)o(t)c(c)o(hanges)f(in)h(the)f(underlying)i (comm)o(unication)f(and)g(system)e(soft)o(w)o(are.)143 2312 y FA(\017)23 b FC(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h (should)g(b)q(e)g(language)f(indep)q(enden)o(t.)143 2403 y FA(\017)23 b FC(The)15 b(in)o(terface)h(should)g(b)q(e)g(designed)g (to)f(allo)o(w)g(for)g(thread-safet)o(y)l(.)75 2546 y Ft(1.2)59 b(Who)20 b(Should)g(Use)g(This)f(Standa)n(rd?)75 2647 y FC(This)d(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h(all) g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 2704 y(programs)9 b(in)j(F)l(ortran)d(77)i(and)f (C.)h(This)g(includes)i(individual)h(application)e(programmers,)e(dev)o (elop)q(ers)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 11 3 10 bop 75 -100 a Fs(1.3.)34 b(WHA)l(T)15 b(PLA)l(TF)o(ORMS)h(ARE)g(T) l(AR)o(GETS)f(F)o(OR)g(IMPLEMENT)l(A)l(TION?)296 b FC(3)75 45 y(of)17 b(soft)o(w)o(are)e(designed)j(to)f(run)g(on)g(parallel)i (mac)o(hines,)f(and)f(creators)f(of)h(en)o(vironmen)o(ts)g(and)g(to)q (ols.)75 102 y(In)g(order)f(to)f(b)q(e)i(attractiv)o(e)e(to)h(this)g (wide)i(audience,)f(the)g(standard)e(m)o(ust)h(pro)o(vide)h(a)e (simple,)j(easy-)75 158 y(to-use)f(in)o(terface)g(for)f(the)g(basic)i (user)f(while)h(not)e(seman)o(tically)i(precluding)h(the)e(high-p)q (erformance)75 214 y(message-passing)e(op)q(erations)h(a)o(v)m(ailable) h(on)e(adv)m(anced)h(mac)o(hines.)75 359 y Ft(1.3)59 b(What)20 b(Platfo)n(rms)g(Are)g(T)-5 b(a)n(rgets)19 b(F)n(o)n(r)i(Implementat)o(ion?)75 461 y FC(The)c(attractiv)o(eness)f (of)g(the)h(message-passing)g(paradigm)f(at)g(least)h(partially)h (stems)e(from)g(its)h(wide)75 518 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o(y)f(ma)o(y)g(run)h(on)g (distributed-memory)h(m)o(ultipro)q(cessors,)75 574 y(net)o(w)o(orks)i (of)i(w)o(orkstations,)e(and)i(com)o(binations)g(of)g(all)g(of)f (these.)34 b(In)20 b(addition,)h(shared-memory)75 631 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 687 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 744 y(th)o(us)d(should)i(b)q(e)f(b)q(oth)g (p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f(on) h(a)f(great)g(v)m(ariet)o(y)g(of)75 800 y(mac)o(hines,)i(including)h (those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h(of)e(other)h (mac)o(hines,)g(parallel)75 857 y(or)d(not,)f(connected)i(b)o(y)f(a)g (comm)o(unication)h(net)o(w)o(ork.)166 913 y(The)21 b(in)o(terface)f (is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i(general)f(MIMD)f (programs,)g(as)g(w)o(ell)h(as)f(those)75 970 y(written)d(in)g(the)g (more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g(Although)h(no)f(explicit)j (supp)q(ort)e(for)f(threads)g(is)75 1026 y(pro)o(vided,)e(the)f(in)o (terface)h(has)f(b)q(een)h(designed)g(so)f(as)g(not)f(to)h(prejudice)i (their)e(use.)20 b(With)13 b(this)h(v)o(ersion)75 1083 y(of)h(MPI)g(no)g(supp)q(ort)g(is)h(pro)o(vided)g(for)f(dynamic)h(spa)o (wning)f(of)g(tasks.)75 1228 y Ft(1.4)59 b(What)20 b(Is)f(Included)g (In)g(The)g(Standa)n(rd?)75 1330 y FC(The)c(standard)g(includes:)143 1437 y FA(\017)23 b FC(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 1532 y FA(\017)23 b FC(Collectiv)o(e)16 b(op)q(erations)143 1627 y FA(\017)23 b FC(Pro)q(cess)15 b(groups)143 1722 y FA(\017)23 b FC(Comm)o(unication)15 b(con)o(texts)143 1817 y FA(\017)23 b FC(Pro)q(cess)15 b(top)q(ology)143 1912 y FA(\017)23 b FC(Bindings)17 b(for)d(F)l(ortran)g(77)h(and)g(C)143 2007 y FA(\017)23 b FC(En)o(vironmen)o(tal)15 b(Managemen)o(t)f(and)i(inquiry)143 2102 y FA(\017)23 b FC(Pro\014ling)16 b(in)o(terface)143 2197 y FA(\017)23 b FC(Subset)15 b(sp)q(eci\014cation)75 2342 y Ft(1.5)59 b(What)20 b(Is)f(Not)h(Included)e(In)i(The)f(Standa)n (rd?)75 2444 y FC(The)c(standard)g(do)q(es)h(not)f(sp)q(ecify:)143 2552 y FA(\017)23 b FC(Explicit)17 b(shared-memory)e(op)q(erations)143 2647 y FA(\017)23 b FC(Op)q(erations)c(that)e(require)i(more)f(op)q (erating)h(system)f(supp)q(ort)g(than)g(is)h(curren)o(tly)g(standard;) 189 2704 y(for)14 b(example,)i(in)o(terrupt-driv)o(en)g(receiv)o(es,)g (remote)f(execution,)h(or)e(activ)o(e)i(messages)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 12 4 11 bop 75 -100 a FC(4)903 b Fs(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION) 17 b(TO)e(MPI)143 45 y FA(\017)23 b FC(Program)13 b(construction)j(to)q (ols)143 135 y FA(\017)23 b FC(Debugging)15 b(facilities)143 225 y FA(\017)23 b FC(Auxiliary)17 b(functions)e(suc)o(h)h(as)f(timers) 143 315 y FA(\017)23 b FC(Explicit)17 b(supp)q(ort)e(for)g(threads)143 405 y FA(\017)23 b FC(Supp)q(ort)15 b(for)g(task)f(managemen)o(t)143 494 y FA(\017)23 b FC(I/O)15 b(functions)166 587 y(There)g(are)f(man)o (y)h(features)f(that)g(ha)o(v)o(e)g(b)q(een)i(considered)g(and)f(not)g (included)i(in)f(this)f(standard.)75 643 y(This)j(happ)q(ened)h(for)e (a)g(n)o(um)o(b)q(er)h(of)f(reasons,)g(one)g(of)g(whic)o(h)i(is)f(the)f (time)h(constrain)o(t)f(that)g(w)o(as)f(self)75 700 y(imp)q(osed)g(in)h (\014nishing)g(the)e(standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g (included)j(can)e(alw)o(a)o(ys)e(b)q(e)i(o\013ered)f(as)75 756 y(extensions)i(b)o(y)f(sp)q(eci\014c)j(implemen)o(tations.)24 b(P)o(erhaps)17 b(future)f(v)o(ersions)g(of)g(MPI)h(will)h(address)e (some)75 813 y(of)f(these)g(issues.)75 954 y Ft(1.6)59 b(Organization)20 b(of)g(this)f(Do)r(cument)75 1056 y FC(The)f(follo)o(wing)h(is)f(a)g(list)g(of)g(the)g(c)o(hapters)f(in)i (this)f(do)q(cumen)o(t,)h(along)f(with)g(a)f(brief)i(description)g(of) 75 1112 y(eac)o(h.)143 1205 y FA(\017)k FC(Chapter)c(2,)i Fv(MPI)e(T)l(erms)h(and)g(Conventions)p FC(,)j(explains)e(notational)f (terms)g(and)g(con)o(v)o(en)o(tions)189 1261 y(used)15 b(throughout)g(the)g(MPI)g(do)q(cumen)o(t.)143 1351 y FA(\017)23 b FC(Chapter)16 b(3,)g Fv(P)o(oint)h(to)f(P)o(oint)h (Communication)p FC(,)f(de\014nes)h(the)g(basic,)g(pairwise)g(comm)o (unication)189 1407 y(subset)c(of)h(MPI.)f Fr(send)g FC(and)h Fr(r)n(e)n(c)n(eive)e FC(are)h(found)h(here,)g(along)g(with)g (man)o(y)f(asso)q(ciated)h(functions)189 1464 y(designed)i(to)f(mak)o (e)f(basic)i(comm)o(unication)g(p)q(o)o(w)o(erful)g(and)f(e\016cien)o (t.)143 1554 y FA(\017)23 b FC(Chapter)12 b(4,)h Fv(Collective)g (Communications)p FC(,)f(de\014nes)i(pro)q(cess-group)f(collectiv)o(e)h (comm)o(unication)189 1610 y(op)q(erations.)28 b(W)l(ell)19 b(kno)o(wn)f(examples)h(of)e(this)i(are)e(barrier)i(and)f(broadcast)f (o)o(v)o(er)g(a)h(group)f(of)189 1667 y(pro)q(cesses)e(\(not)g (necessarily)h(all)g(the)g(pro)q(cesses\).)143 1756 y FA(\017)23 b FC(Chapter)16 b(5,)g Fv(Groups,)h(Contexts,)i(and)e (Communicato)o(rs)p FC(,)e(sho)o(ws)h(ho)o(w)g(groups)g(of)g(pro)q (cesses)h(are)189 1813 y(formed)c(and)h(manipulated,)i(ho)o(w)d(unique) i(comm)o(unication)g(con)o(texts)e(are)h(obtained,)g(and)g(ho)o(w)189 1869 y(the)h(t)o(w)o(o)f(are)h(b)q(ound)h(together)e(in)o(to)h(a)g Fr(c)n(ommunic)n(ator)p FC(.)143 1959 y FA(\017)23 b FC(Chapter)18 b(6,)h Fv(Pro)q(cess)h(T)l(op)q(ologies)p FC(,)f(explains)h(a)f(set)f(of)h(utilit)o(y)h(functions)f(mean)o(t)f (to)h(assist)f(in)189 2016 y(the)13 b(mapping)i(of)e(pro)q(cess)g (groups)h(\(a)f(linearly)i(ordered)f(set\))f(to)g(ric)o(her)h(top)q (ological)g(structures)189 2072 y(suc)o(h)h(as)g(m)o(ulti-dimensional)j (grids.)143 2162 y FA(\017)23 b FC(Chapter)10 b(7,)h Fv(MPI)g(Environmental)g(Management)p FC(,)g(explains)h(ho)o(w)f(the)g (programmer)e(can)i(manage)189 2219 y(and)22 b(mak)o(e)f(inquiries)j (of)d(the)h(curren)o(t)f(MPI)h(en)o(vironmen)o(t.)40 b(These)22 b(functions)g(are)g(needed)189 2275 y(for)g(the)h(writing)g (of)f(correct,)i(robust)e(programs,)i(and)f(are)f(esp)q(ecially)j(imp)q (ortan)o(t)e(for)f(the)189 2331 y(construction)15 b(of)g(highly-p)q (ortable)i(message-passing)f(programs.)143 2421 y FA(\017)23 b FC(Chapter)c(8,)i Fv(Pro\014ling)f(Interface)p FC(,)h(explains)g(a)f (simple)i(name)e(shifting)h(con)o(v)o(en)o(tion)f(that)f(an)o(y)189 2478 y(MPI)i(implemen)o(tation)i(m)o(ust)e(supp)q(ort.)38 b(One)22 b(motiv)m(ation)g(for)f(this)h(is)g(the)f(abilit)o(y)i(to)e (put)189 2534 y(p)q(erformance)c(pro\014ling)i(calls)f(in)o(to)g(MPI)f (without)h(the)f(need)i(for)d(access)i(to)f(the)g(MPI)h(source)189 2591 y(co)q(de.)31 b(The)19 b(name)g(shift)g(is)h(merely)f(an)g(in)o (terface,)h(it)f(sa)o(ys)f(nothing)i(ab)q(out)e(ho)o(w)h(the)g(actual) 189 2647 y(pro\014ling)h(should)g(b)q(e)f(done)h(and)f(in)h(fact,)f (the)g(name)g(shift)g(can)g(usefully)i(b)q(e)e(used)h(for)e(other)189 2704 y(purp)q(oses.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 13 5 12 bop 75 -100 a Fs(1.6.)34 b(OR)o(GANIZA)l(TION)17 b(OF)e(THIS)h(DOCUMENT)810 b FC(5)143 45 y FA(\017)23 b FC(Chapter)10 b(9,)h Fv(Initial)f(Implementation)g(Subset)p FC(,)j(suggests)d(to)g(MPI)h(implemen)o(tors)g(a)f(\\core")g(subset)189 102 y(of)16 b(MPI)h(that)g(is)g(useful,)h(in)o(ternally)h(consisten)o (t,)e(and)g(should)h(app)q(ear)g(\014rst)e(in)i(the)f(ev)o(olution)189 158 y(of)d(an)h(MPI)g(implemen)o(tation.)21 b(The)15 b(subset)g(is)h(de\014ned)g(so)e(that)h(consisten)o(t)g(implemen)o (tations)189 214 y(can)d(app)q(ear)g(rapidly)l(,)i(and)e(p)q(ortable)h (parallel)h(programming)d(with)i(MPI)f(can)g(start)f(in)i(a)f(timely) 189 271 y(manner.)143 365 y FA(\017)23 b FC(Annex)18 b(A,)f Fv(Language)g(Bindings)p FC(,)i(giv)o(es)f(sp)q(eci\014c)h(syn)o (tax)d(in)i(F)l(ortran)e(77)h(and)h(C,)e(for)h(all)h(MPI)189 421 y(functions,)d(constan)o(ts,)f(and)h(t)o(yp)q(es.)143 515 y FA(\017)23 b FC(The)g Fv(MPI)g(F)o(unction)i(Index)f FC(is)g(a)g(simple)h(index)f(sho)o(wing)g(the)g(lo)q(cation)g(of)f(the) h(language-)189 571 y(indep)q(enden)o(t)17 b(de\014nition)g(of)e(eac)o (h)g(MPI)g(function.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 14 6 13 bop 75 356 a Fu(Chapter)34 b(2)75 564 y Fy(MPI)40 b(T)-10 b(erms)39 b(and)g(Con)m(v)m(en)m(tions)75 786 y FC(This)11 b(c)o(hapter)g(explains)h(notational)e(terms)g(and)h(con)o (v)o(en)o(tions)g(used)g(throughout)f(the)g(MPI)h(do)q(cumen)o(t,)75 843 y(some)k(of)g(the)g(c)o(hoices)h(that)e(ha)o(v)o(e)h(b)q(een)i (made,)d(and)i(the)f(rationale)h(b)q(ehind)h(those)e(c)o(hoices.)75 984 y Ft(2.1)59 b(Pro)r(cedure)19 b(Sp)r(eci\014cation)75 1086 y FC(MPI)g(pro)q(cedures)h(are)f(sp)q(eci\014ed)j(using)e(a)f (language)g(indep)q(enden)o(t)j(notation.)31 b(The)20 b(argumen)o(ts)e(of)75 1142 y(pro)q(cedure)e(calls)g(are)f(mark)o(ed)g (as)g Fq(IN)p FC(,)g Fq(OUT)g FC(or)g Fq(INOUT)p FC(.)g(The)h(meanings) f(of)g(these)h(are:)143 1226 y FA(\017)23 b FC(the)15 b(call)h(uses)g(but)f(do)q(es)h(not)e(up)q(date)i(an)f(argumen)o(t)g (mark)o(ed)f Fq(IN)p FC(,)143 1315 y FA(\017)23 b FC(the)15 b(call)h(ma)o(y)f(up)q(date)h(an)f(argumen)o(t)f(mark)o(ed)h Fq(OUT)p FC(,)143 1405 y FA(\017)23 b FC(the)15 b(call)h(b)q(oth)g (uses)f(and)g(up)q(dates)h(an)f(argumen)o(t)g(mark)o(ed)f Fq(INOUT)p FC(.)166 1489 y(There)j(is)h(one)f(sp)q(ecial)h(case)f(|)h (if)f(an)g(argumen)o(t)f(is)h(a)g(handle)h(to)e(an)h(opaque)g(ob)s (ject,)g(and)g(the)75 1545 y(ob)s(ject)g(is)h(up)q(dated)g(b)o(y)f(the) h(pro)q(cedure)g(call,)h(then)e(the)h(argumen)o(t)f(is)g(mark)o(ed)g Fq(OUT)p FC(.)h(This)g(is)g(done)75 1602 y(ev)o(en)f(though)g(the)g (handle)h(itself)f(is)h(not)e(mo)q(di\014ed)i(|)f(w)o(e)g(use)g(the)g Fq(OUT)g FC(attribute)g(to)f(denote)h(that)75 1658 y(what)e(the)g (handle)h Fr(r)n(efer)n(enc)n(es)e FC(is)h(up)q(dated.)166 1715 y(The)h(de\014nition)j(of)c(MPI)i(tries)f(to)g(a)o(v)o(oid,)g(to)f (the)i(largest)f(p)q(ossible)i(exten)o(t,)e(the)g(use)h(of)e Fq(INOUT)75 1771 y FC(argumen)o(ts,)f(b)q(ecause)i(suc)o(h)g(use)f(is)h (error-prone,)e(esp)q(ecially)k(for)d(scalar)g(argumen)o(ts.)166 1828 y(A)d(common)f(o)q(ccurrence)i(for)f(MPI)f(functions)i(is)f(an)g (argumen)o(t)f(that)g(is)i(used)f(as)g Fq(IN)g FC(b)o(y)g(some)f(pro-) 75 1884 y(cesses)h(and)h Fq(OUT)f FC(b)o(y)g(other)g(pro)q(cesses.)19 b(Suc)o(h)13 b(argumen)o(ts)e(are,)h(syn)o(tactically)l(,)i(an)e Fq(INOUT)g FC(argumen)o(t,)75 1940 y(although,)j(seman)o(tically)l(,)h (they)f(are)g(not)g(used)h(in)g(one)f(call)i(b)q(oth)e(for)g(input)h (and)f(for)g(output.)166 1997 y(Another)h(frequen)o(t)g(situation)h (arises)g(when)f(an)g(argumen)o(t)g(v)m(alue)h(is)g(needed)h(only)f(b)o (y)f(a)g(subset)75 2053 y(of)f(the)g(pro)q(cesses.)20 b(When)c(an)f(argumen)o(t)f(is)i(not)f(signi\014can)o(t)h(at)e(a)h(pro) q(cess)h(then)f(an)g(arbitrary)g(v)m(alue)75 2110 y(can)g(b)q(e)h (passed)g(as)e(argumen)o(t.)75 2251 y Ft(2.2)59 b(Semantic)18 b(T)-5 b(erms)75 2353 y FC(When)12 b(discussing)i(p)q(oin)o(t-to-p)q (oin)o(t)f(and)f(collectiv)o(e)h(comm)o(unications)g(the)f(follo)o (wing)h(terms)e(are)h(used:)75 2445 y Fx(non)o(blo)q(c)o(king)24 b FC(if)e(the)h(pro)q(cedure)g(ma)o(y)f(return)g(b)q(efore)h(the)f (related)h(comm)o(unication)g(op)q(eration)189 2501 y(completes,)13 b(and)g(b)q(efore)f(the)h(user)f(is)h(allo)o(w)o(ed)g(to)f(re-use)h (resources)f(\(suc)o(h)h(as)f(bu\013ers\))g(sp)q(eci\014ed)189 2557 y(in)k(the)f(call;)75 2647 y Fx(lo)q(cally-blo)r(c)o(king)26 b FC(if)18 b(return)g(from)f(the)h(pro)q(cedure)h(indicates)g(the)f (user)g(is)h(allo)o(w)o(ed)f(to)f(re-use)i(re-)189 2704 y(sources)c(sp)q(eci\014ed)i(in)f(the)f(call;)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 15 7 14 bop 75 -100 a Fs(2.3.)34 b(D)o(A)l(T)l(A)15 b(TYPES)1375 b FC(7)75 45 y Fx(globally-blo)q(cking)25 b FC(if)c(return)f(from)g (the)g(pro)q(cedure)i(indicates)f(the)g(comm)o(unication)g(op)q (eration)189 102 y(has)15 b(completed.)166 217 y(When)i(discussing)i (the)e(b)q(eha)o(vior)h(of)e(an)o(y)h(MPI)g(function,)h(not)e (necessarily)j(a)d(comm)o(unication)75 273 y(function,)g(the)f(follo)o (wing)h(terms)f(are)f(used.)75 388 y Fx(lo)q(cal)25 b FC(An)16 b(op)q(eration)g(is)g Fx(lo)q(cal)i FC(if)e(its)g(completion)h (dep)q(ends)h(only)e(on)g(the)f(lo)q(cal)i(executing)g(pro)q(cess.)189 444 y(Suc)o(h)e(an)h(op)q(eration)f(do)q(es)h(not)e(require)i(comm)o (unication)g(with)g(another)f(user)g(pro)q(cess.)75 545 y Fx(non-lo)q(cal)26 b FC(An)c(op)q(eration)g(is)g Fx(non-lo)q(cal)j FC(if)d(its)g(completion)h(ma)o(y)e(dep)q(end)i(on)f(comm)o(unication) 189 602 y(o)q(ccurring)16 b(with)f(another)g(user)g(pro)q(cess.)75 702 y Fx(collectiv)o(e)24 b FC(An)17 b(op)q(eration)g(is)g Fx(collectiv)o(e)i FC(if)e(all)g(pro)q(cesses)g(in)h(a)e(pro)q(cess)h (group)g(need)g(to)f(p)q(erform)189 759 y(the)f(op)q(eration.)75 912 y Ft(2.3)59 b(Data)19 b(T)-5 b(yp)r(es)75 1019 y Fp(2.3.1)49 b(Opaque)16 b(objects)75 1108 y FC(MPI)23 b(pro)q(cedures)g(use)g Fr(op)n(aque)h(obje)n(cts)p FC(.)42 b(These)23 b(hide)h(the)f(in)o(ternal)h(represen)o(tation)e(used,)j(th) o(us)75 1164 y(allo)o(wing)17 b(similar)h(calls)f(in)h(C)e(and)g(F)l (ortran;)g(the)g(a)o(v)o(oidance)h(of)f(con\015icts)h(with)g(the)f(t)o (yping)h(rules)g(in)75 1221 y(these)f(languages;)h(and)f(future)h (extension)g(of)f(functionalit)o(y)l(.)24 b(The)17 b(mec)o(hanism)g (for)e(opaque)i(ob)s(jects)75 1277 y(used)f(here)f(lo)q(osely)i(follo)o (ws)e(the)g(POSIX)h(F)l(ortran)e(binding)k(standard.)166 1335 y(An)g(opaque)g(ob)s(ject)f(can)h(b)q(e)h(allo)q(cated)g(in)g (user)f(storage)e(or)i(in)g(system)g(\(i.e.)28 b(MPI)18 b(managed\))75 1392 y(storage.)23 b(W)l(e)17 b(call)h(the)f(former)f(a) g Fr(user)i(obje)n(ct)e FC(and)h(the)g(latter)g(a)f Fr(system)h(obje)n (ct)p FC(.)24 b(User)17 b(ob)s(jects)f(are)75 1448 y(allo)q(cated)h (and)f(deallo)q(cated)h(explicitly)h(b)o(y)e(the)g(user)g(as)f(a)h (regular)f(v)m(ariable.)23 b(The)16 b(t)o(yp)q(e)g(de\014nitions)75 1505 y(for)g(suc)o(h)h(ob)s(jects)f(are)h(pro)o(vided)g(b)o(y)g(MPI.)f (The)h(ob)s(jects)f(can)h(b)q(e)g(accessed)g(and)g(mo)q(di\014ed)h (only)g(via)75 1561 y(MPI)d(functions)h(that)f(tak)o(e)f(suc)o(h)i(ob)s (jects)e(as)h(argumen)o(ts.)166 1619 y(System)d(ob)s(jects)g(are)g (accessed)i(only)f(via)f Fr(hand)r(les)p FC(.)19 b(In)13 b(C,)f(a)g(di\013eren)o(t)h(t)o(yp)q(e)g(of)f(handle)i(is)f(de\014ned) 75 1676 y(b)o(y)i(MPI)g(for)f(eac)o(h)h(t)o(yp)q(e)g(of)g(opaque)g (system)f(ob)s(ject.)20 b(In)15 b(F)l(ortran,)f(all)i(handles)g(ha)o(v) o(e)e(t)o(yp)q(e)h Fo(INTEGER)p FC(.)75 1732 y(The)g(allo)q(cation)g (and)g(deallo)q(cation)h(of)e(suc)o(h)h(an)f(ob)s(ject,)g(and)g(an)o(y) h(access)f(or)g(up)q(date)h(to)f(it,)g(are)h(done)75 1789 y(only)h(via)f(MPI)g(calls)i(that)d(accept)i(as)e(argumen)o(ts)h (handles)h(to)f(suc)o(h)g(ob)s(jects.)166 1847 y(An)f(opaque)f(ob)s (ject,)g(and)h(the)g(handle)g(to)f(suc)o(h)h(an)g(ob)s(ject,)e(are)i (signi\014can)o(t)g(only)g(at)f(the)h(pro)q(cess)75 1903 y(where)h(the)h(ob)s(ject)e(w)o(as)h(created,)g(and)g(cannot)g(b)q(e)h (transferred)f(to)f(another)h(pro)q(cess.)166 1962 y(Opaque)h(system)e (ob)s(jects)h(are)g(allo)q(cated)h(and)f(deallo)q(cated)i(b)o(y)e (calls)h(that)f(are)f(sp)q(eci\014c)j(to)e(eac)o(h)75 2018 y(ob)s(ject)k(t)o(yp)q(e.)32 b(These)20 b(are)f(listed)i(in)f(the) f(sections)h(where)g(the)f(ob)s(jects)g(are)g(describ)q(ed.)35 b(A)19 b(call)i(to)75 2075 y(deallo)q(cate)15 b(an)f(opaque)h(system)e (ob)s(ject)h(do)q(es)g(not)g(necessarily)h(deallo)q(cate)h(the)e(ob)s (ject)f(immediately)l(,)75 2131 y(but)i(ma)o(y)e(only)i(mark)f(it)h(as) f(candidate)h(for)f(deallo)q(cation.)21 b(The)14 b(ob)s(ject)g(will)i (b)q(e)f(deallo)q(cated)h(as)e(so)q(on)75 2187 y(as)k(there)g(is)g(no)g (p)q(ending)i(op)q(eration)f(that)e(is)h(using)h(this)g(ob)s(ject;)f (the)g(handle)i(then)e(b)q(ecomes)h(n)o(ull.)75 2244 y(F)l(or)13 b(example,)h(if)f(a)g(comm)o(unication)h(ob)s(ject)e(is)i (freed)g(after)e(the)h(comm)o(unication)h(started)e(but)i(b)q(efore)75 2300 y(it)19 b(completed,)g(then)g(the)g(p)q(ending)h(op)q(eration)e (is)h(not)f(a\013ected;)i(one)e(can)h(use)g(the)f(handle)i(in)f(calls) 75 2357 y(to)c Fv(MPI)p 216 2357 14 2 v 15 w(W)l(AIT)h FC(or)e Fv(MPI)p 497 2357 V 16 w(ST)l(A)l(TUS)p FC(.)i(In)g(fact,)e (one)i(m)o(ust)e(complete)i(the)f(comm)o(unication)h(with)g(suc)o(h)f (a)75 2413 y(call.)25 b(Ho)o(w)o(ev)o(er,)15 b(once)i(the)f(op)q (eration)h(completes,)g(it)g(is)g(erroneous)f(to)g(re-use)h(the)f (freed)h(handle)h(for)75 2470 y(a)d(new)g(comm)o(unication.)166 2604 y Fn(Implemen)o(tati)o(on)g(note:)47 b Fq(Some)15 b(ob)r(jects,)j(suc)o(h)f(as)g(datat)o(yp)q(e)f(ob)r(jects)i(\(see)g (section)f(3.13\),)f(can)g(b)q(e)75 2654 y(used)f(b)o(y)f(sev)o(eral,)g (distinct,)g(concurren)o(t)i(comm)o(unicati)o(on)11 b(op)q(erations.)19 b(Th)o(us,)14 b(to)g(\014nd)g(out)g(whether)h(suc)o(h)g(an)75 2704 y(ob)r(ject)g(can)f(b)q(e)g(freed,)h(one)f(needs)h(a)e(reference)k (coun)o(t)d(of)f(p)q(ending)h(op)q(erations.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 8 16 8 15 bop 75 -100 a FC(8)715 b Fs(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)166 45 y Fq(MPI)c(do)q(es)h(not)f(k)o(eep)h (coun)o(t)f(of)f(references)k(to)d(an)g(opaque)g(MPI)g(ob)r(ject)h (that)f(are)h(created)g(b)o(y)f(assigning)f(to)75 102 y(one)j(handle)f(the)h(v)n(alue)e(of)h(another,)h(or)f(are)h(lost)f (when)h(a)f(handle)g(v)n(ariable)f(is)i(deallo)q(cated)f(\()p Fm(e.g.)p Fq(,)g(at)g(pro)q(cedure)75 158 y(exit\).)22 b(In)16 b(the)f(\014rst)h(case,)h(a)d(call)h(to)g(free)h(the)g(ob)r (ject)g(ma)o(y)d(lea)o(v)o(e)i(a)g(dangling)f(reference;)k(in)d(the)h (second)g(case)75 214 y(an)f(opaque)g(MPI)g(ob)r(ject)h(ma)o(y)d(b)q(e) j(left)f(with)f(no)h(handle)g(to)g(it,)g(and)g(no)f(p)q(ossibilit)o(y)g (of)h(deallo)q(cating)f(it.)21 b(It)15 b(is)75 271 y(the)h(user's)g (resp)q(onsibilit)o(y)f(to)g(a)o(v)o(oid)f(these)j(situations.)k(MPI)16 b(k)o(eeps)g(trac)o(k)g(only)e(of)h(c)o(hanges)h(in)f(the)h(n)o(um)o(b) q(er)75 327 y(of)d(activ)o(e)h(references)j(to)d(the)g(MPI)g(ob)r(ject) h(that)f(are)g(a)g(result)g(of)g(explicit)f(MPI)h(calls.)166 468 y FC(MPI)19 b(ma)o(y)g(pro)o(vide)h(prede\014ned)h(opaque)f(ob)s (jects)f(and)g(prede\014ned,)j(static)d(handles)i(to)e(these)75 525 y(ob)s(jects.)g(Suc)o(h)d(ob)s(jects)f(ma)o(y)f(not)h(b)q(e)h (destro)o(y)o(ed.)75 656 y Fp(2.3.2)49 b(Arra)o(y)16 b(a)o(rguments)75 745 y FC(An)21 b(MPI)f(call)i(ma)o(y)e(need)h(an)f (argumen)o(t)g(that)g(is)h(an)f(arra)o(y)f(of)h(opaque)h(ob)s(jects,)g (or)f(an)g(arra)o(y)f(of)75 802 y(handles.)31 b(The)19 b(arra)o(y-of-handles)f(is)h(a)f(regular)g(arra)o(y)g(with)g(en)o (tries)h(that)f(are)g(handles)i(to)e(ob)s(jects)75 858 y(of)g(the)h(same)g(t)o(yp)q(e)f(in)i(consecutiv)o(e)g(lo)q(cations)f (in)h(the)f(arra)o(y)l(.)29 b(Whenev)o(er)19 b(suc)o(h)g(an)g(arra)o(y) e(is)j(used,)75 915 y(an)d(additional)h Fv(len)f FC(argumen)o(t)f(is)h (required)h(to)e(indicate)i(the)f(n)o(um)o(b)q(er)g(of)f(v)m(alid)j(en) o(tries)e(\(unless)h(this)75 971 y(n)o(um)o(b)q(er)f(can)h(b)q(e)g (deriv)o(ed)g(otherwise\).)25 b(The)18 b(v)m(alid)h(en)o(tries)e(are)g (at)g(the)g(b)q(egining)i(of)e(the)g(arra)o(y;)f Fv(len)75 1028 y FC(indicates)j(ho)o(w)e(man)o(y)h(of)f(them)h(there)g(are,)f (and)h(need)h(not)e(b)q(e)h(the)g(en)o(tire)g(size)h(of)e(the)h(arra)o (y)l(.)27 b(The)75 1084 y(same)15 b(approac)o(h)g(is)g(follo)o(w)o(ed)h (for)f(other)f(arra)o(y)g(argumen)o(ts.)75 1216 y Fp(2.3.3)49 b(State)75 1305 y FC(MPI)15 b(pro)q(cedures)h(use)g(at)e(v)m(arious)i (places)g(argumen)o(ts)f(with)g Fr(state)g FC(t)o(yp)q(es.)20 b(The)c(v)m(alues)g(of)f(suc)o(h)h(data)75 1361 y(t)o(yp)q(e)i(are)h (all)g(iden)o(ti\014ed)h(b)o(y)f(names,)f(and)h(no)f(op)q(eration)h(is) g(de\014ned)h(on)e(them.)29 b(F)l(or)18 b(example,)i(the)75 1418 y Fv(MPI)p 160 1418 14 2 v 16 w(SET)p 259 1418 V 17 w(ERRORMODE)c FC(routine)f(has)f(a)g(state)g(t)o(yp)q(e)h(argumen)o (t)e(with)i(v)m(alues)h Fl(MPI)p 1572 1418 13 2 v 14 w(ERRORSF)m(A)m(T)m(AL)p FC(,)75 1474 y Fl(MPI)p 152 1474 V 14 w(ERRORSRETURN)p FC(,)e(etc.)75 1606 y Fp(2.3.4)49 b(Named)16 b(constants)75 1695 y FC(MPI)e(pro)q(cedures)g(sometimes)g (assign)f(a)h(sp)q(ecial)h(meaning)f(to)f(a)h(sp)q(ecial)h(v)m(alue)g (of)e(a)g(basic)i(t)o(yp)q(e)e(argu-)75 1751 y(men)o(t;)k(e.g.)24 b Fv(tag)17 b FC(is)g(an)g(in)o(teger)g(v)m(alued)i(argumen)o(t)d(of)g (p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)f(op)q(erations,)75 1808 y(with)f(a)g(sp)q(ecial)i(wild-card)f(v)m(alue,)g Fl(MPI)p 775 1808 V 15 w(ANY)p 875 1808 V 14 w(T)m(A)o(G)p FC(.)e(Suc)o(h)i(argumen)o(ts)e(will)i(ha)o(v)o(e)f(a)g(range)f(of)h (regular)75 1864 y(v)m(alues,)i(whic)o(h)g(is)f(a)f(prop)q(er)h (subrange)g(of)f(the)h(range)g(of)f(v)m(alues)i(of)e(the)h(corresp)q (onding)h(basic)f(t)o(yp)q(e;)75 1921 y(sp)q(ecial)f(v)m(alues)g(\(suc) o(h)f(as)f Fl(MPI)p 609 1921 V 14 w(ANY)p 708 1921 V 15 w(T)m(A)o(G)p FC(\))f(will)j(b)q(e)f(outside)g(the)g(regular)g (range.)k(The)c(range)f(of)g(regu-)75 1977 y(lar)f(v)m(alues)i(can)e(b) q(e)h(queried,)g(and)f(sometimes)g(set,)g(using)h(en)o(vironmen)o(tal)g (inquiry)h(or)d(en)o(vironmen)o(tal)75 2034 y(managemen)o(t)i (functions)i(\(Chapter)f(7\).)166 2174 y Fn(Implemen)o(tati)o(on)d (note:)35 b Fq(Implemen)o(tors)11 b(should)i(detect,)i(whenev)o(er)g(p) q(ossible,)e(illegal)e(uses)k(of)d(\\sp)q(e-)75 2231 y(cial)i(v)n(alues".)22 b(Th)o(us,)16 b(the)f(use)i(of)d(the)i Fl(MPI)p 780 2231 V 14 w(ANY)p 879 2231 V 15 w(T)m(A)o(G)e Fq(v)n(alue)h(to)g(tag)g(a)g(message)g(sen)o(t)h(should)f(b)q(e)h (\015agged)f(as)75 2287 y(an)f(error.)75 2502 y Fp(2.3.5)49 b(Choice)75 2591 y FC(MPI)11 b(functions)h(sometimes)f(use)h(argumen)o (ts)e(with)h(a)g Fr(choic)n(e)g FC(\(or)g(union\))g(data)g(t)o(yp)q(e.) 19 b(Distinct)11 b(calls)i(to)75 2647 y(the)f(same)g(routine)g(ma)o(y)g (pass)f(b)o(y)h(reference)h(actual)f(argumen)o(ts)f(of)h(di\013eren)o (t)g(t)o(yp)q(es.)19 b(The)12 b(mec)o(hanism)75 2704 y(for)j(pro)o(viding)h(suc)o(h)f(argumen)o(ts)g(will)h(di\013er)g(from) e(language)i(to)e(language.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 9 17 9 16 bop 75 -100 a Fs(2.4.)34 b(LANGUA)o(GE)15 b(BINDING)1187 b FC(9)75 45 y Fp(2.3.6)49 b(Addresses)75 134 y FC(Some)16 b(MPI)g(pro)q(cedures)h(use)f Fr(addr)n(ess)f FC(argumen)o(ts)h(that)f (represen)o(t)h(an)g(address)g(in)g(the)g(calling)i(pro-)75 191 y(gram.)g(The)c(datat)o(yp)q(e)e(of)h(suc)o(h)h(an)f(argumen)o(t)f (is)i(an)f(in)o(teger)g(of)g(the)h(size)g(needed)g(to)f(hold)h(an)f (address)75 247 y(in)j(the)f(execution)i(en)o(vironmen)o(t.)75 400 y Ft(2.4)59 b(Language)19 b(Binding)75 505 y FC(This)k(section)g (de\014nes)h(the)e(rules)i(for)e(MPI)g(language)h(binding)h(in)g (general)f(and)f(for)g(F)l(ortran)g(77)75 562 y(and)17 b(ANSI)h(C)f(in)h(particular.)26 b(De\014ned)17 b(here)h(are)f(v)m (arious)g(ob)s(ject)g(represen)o(tations,)g(as)f(w)o(ell)i(as)f(the)75 618 y(naming)k(con)o(v)o(en)o(tions)f(used)g(for)g(expressing)h(this)f (standard.)35 b(The)20 b(actual)g(calling)i(sequences)f(are)75 674 y(de\014ned)c(elsewhere.)166 733 y(It)23 b(is)g(exp)q(ected)h(that) e(an)o(y)g(F)l(ortran)g(90)g(and)h(C++)g(implemen)o(tations)h(use)f (the)g(F)l(ortran)f(77)75 789 y(and)f(ANSI)h(C)e(bindings,)k(resp)q (ectiv)o(ely)l(.)39 b(Although)22 b(w)o(e)e(consider)i(it)f(premature)g (to)f(de\014ne)i(other)75 846 y(bindings)e(to)d(F)l(ortran)g(90)g(and)h (C++,)h(the)f(curren)o(t)f(bindings)j(are)e(designed)h(to)e(encourage,) i(rather)75 902 y(than)c(discourage,)g(exp)q(erimen)o(tation)i(with)e (b)q(etter)g(bindings)i(whic)o(h)f(migh)o(t)f(b)q(e)h(adopted)g(later.) 166 960 y(Since)h(the)e(w)o(ord)g(P)l(ARAMETER)h(is)g(a)f(k)o(eyw)o (ord)g(in)h(the)g(F)l(ortran)e(language,)h(w)o(e)g(use)h(the)g(w)o(ord) 75 1017 y(\\argumen)o(t")21 b(to)i(denote)g(the)g(argumen)o(ts)e(to)i (a)f(subroutine.)44 b(These)23 b(are)f(normally)i(referred)f(to)75 1073 y(as)f(parameters)f(in)i(C,)e(ho)o(w)o(ev)o(er)g(w)o(e)h(exp)q (ect)h(that)e(C)h(programmers)e(will)k(understand)f(the)f(w)o(ord)75 1130 y(argumen)o(ts)d(\(whic)o(h)i(has)g(no)f(sp)q(eci\014c)i(meaning)f (in)g(C\),)f(th)o(us)g(allo)o(wing)h(us)f(to)g(a)o(v)o(oid)g (unnecessary)75 1186 y(confusion)c(for)f(F)l(ortran)f(programmers.)166 1244 y(There)k(are)g(sev)o(eral)g(imp)q(ortan)o(t)f(language)h(binding) i(issues)f(not)e(addressed)i(b)o(y)e(this)i(standard.)75 1301 y(It)f(is)g(b)q(ey)o(ond)h(the)f(scop)q(e)g(of)g(this)g(standard)g (to)f(discuss)i(the)f(in)o(terop)q(erabilit)o(y)i(of)d(message)h (passing)75 1357 y(b)q(et)o(w)o(een)c(languages.)19 b(Although)c(it)e (w)o(ould)i(b)q(e)f(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)h(to)e (guaran)o(tee)g(that)f(messages)75 1414 y(sen)o(t)20 b(b)q(et)o(w)o(een)h(languages)f(b)q(e)h(prop)q(erly)g(in)o(terpreted,) h(the)e(matc)o(hing)g(of)g(t)o(yp)q(es)g(across)g(languages)75 1470 y(prev)o(en)o(ts)15 b(the)h(inclusion)i(of)d(suc)o(h)h(a)f(guaran) o(tee)g(in)i(this)f(standard.)k(It)c(is)g(fully)h(exp)q(ected)g(that)d (man)o(y)75 1526 y(implemen)o(tations)i(will)g(ha)o(v)o(e)f(suc)o(h)g (features,)f(and)h(that)f(suc)o(h)h(features)g(are)f(a)h(sign)g(of)f (the)h(qualit)o(y)g(of)75 1583 y(the)g(implemen)o(tation.)75 1714 y Fp(2.4.1)49 b(Data)17 b(T)l(yp)q(es)f(and)h(Naming)g (Conventions)75 1804 y FC(Handles)e(are)f(represen)o(ted)h(as)e(in)o (tegers)i(in)g(F)l(ortran)e(and)h(as)g(p)q(oin)o(ters)g(to)g (structures)f(in)i(C.)f(A)g(sp)q(ecial)75 1860 y(v)m(alue)g Fl(MPI)p 267 1860 13 2 v 14 w(NULL)g FC(is)f(used)g(to)g(represen)o(t)f (n)o(ull)j(handles)e(in)h(F)l(ortran;)e(in)i(C,)e(the)h(constan)o(t)f Fl(NULL)h FC(is)g(used)75 1917 y(as)i(usual.)166 1975 y(T)o(yp)q(ed)h(comm)o(unication)g(routines)g(supp)q(ort)g Fx(c)o(haracter)p FC(,)g Fx(double)i(precision)p FC(,)e Fx(real)p FC(,)g(and)g Fx(in-)75 2031 y(teger)d FC(data)f(in)h(F)l (ortran.)18 b Fx(in)o(t)p FC(,)13 b Fx(double)p FC(,)h Fx(real)p FC(,)f Fx(short)p FC(,)g Fx(c)o(har)p FC(,)f(and)h Fx(long)h FC(are)e(supp)q(orted)h(in)h(C,)e(along)75 2088 y(with)k Fx(unsigned)j(in)o(t)p FC(,)d Fx(unsigned)i(long)p FC(,)f Fx(unsigned)h(short)p FC(,)e(and)g Fx(unsigned)i(c)o(har)p FC(.)k(In)16 b(addition,)75 2144 y(a)e(sp)q(ecial)j Fx(b)o(yte)d FC(t)o(yp)q(e)g(is)h(supp)q(orted)h(in)f(b)q(oth)g(languages.)k(This)d (t)o(yp)q(e)e(allo)o(ws)h(comm)o(unication)g(to)f(b)q(e)75 2201 y(p)q(erformed)f(with)g(no)f(t)o(yp)q(e)h(con)o(v)o(ersion)f(at)g (all.)20 b(In)13 b(C,)f(the)h(de\014nitions)h(of)f(named)f(constan)o (ts,)g(function)75 2257 y(protot)o(yp)q(es,)k(and)h(aggregate)f(t)o(yp) q(e)g(information)h(m)o(ust)g(b)q(e)g(supplied)j(in)d(an)g(include)i (\014le)f Fo(mpi.h)p FC(.)24 b(In)75 2313 y(F)l(ortran,)c(the)h (corresp)q(onding)h(include)g(\014le)g(for)e(named)h(constan)o(ts)e(\() p Fo(PARAMETER)p FC(s\))g(will)j(b)q(e)f(called)75 2370 y Fo(mpif.h)p FC(.)75 2502 y Fp(2.4.2)49 b(F)o(o)o(rtran)15 b(77)i(Binding)h(Issues)75 2591 y FC(All)13 b(MPI)e(names)h(ha)o(v)o(e) f(an)h Fo(MPI)p 618 2591 15 2 v 28 w FC(pre\014x,)g(and)g(all)g(c)o (haracters)f(are)g(capitals.)20 b(All)13 b(MPI)e(F)l(ortran)g(subrou-) 75 2647 y(tines)16 b(ha)o(v)o(e)g(a)f(return)h(co)q(de)g(in)h(the)f (last)f(argumen)o(t.)21 b(A)16 b(few)g(MPI)f(op)q(erations)h(are)g (functions,)g(whic)o(h)75 2704 y(do)g(not)g(ha)o(v)o(e)f(the)i(return)f (co)q(de)g(argumen)o(t.)22 b(The)16 b(return)g(co)q(de)h(v)m(alue)g (for)f(successful)h(completion)g(is)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 10 18 10 17 bop 75 -100 a FC(10)692 b Fs(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)290 81 y Fo(double)23 b(precision)f(a)290 138 y(integer)h(b)290 194 y(call)g (MPI_send\(a,...\))385 251 y(.)385 307 y(.)290 363 y(call)g (MPI_send\(b,...\))75 511 y FC(Figure)e(2.1:)29 b(This)21 b(is)g(an)f(example)h(of)f(calling)i(a)e(routine)h(with)g(mismatc)o (hed)f(formal)g(and)h(actual)75 568 y(argumen)o(ts.)75 704 y Fl(MPI)p 152 704 13 2 v 14 w(SUCCESS)p FC(;)11 b(error)h(co)q(des)i(are)e(implemen)o(tation)j(dep)q(enden)o(t.)20 b(Unless)14 b(explicitly)i(stated,)c(the)h(MPI)75 760 y(F77)i(binding)k(is)e(consisten)o(t)f(with)h(ANSI)g(standard)f(F)l (ortran)f(77.)23 b(There)16 b(are)g(sev)o(eral)h(p)q(oin)o(ts)g(where) 75 817 y(this)d(standard)f(div)o(erges)g(from)g(the)g(ANSI)h(F)l (ortran)e(77)h(standard.)19 b(These)14 b(exceptions)g(are)f(consisten)o (t)75 873 y(with)j(common)e(practice)i(in)g(the)g(F)l(ortran)e(comm)o (unit)o(y)l(.)20 b(In)15 b(particular:)143 972 y FA(\017)23 b FC(MPI)15 b(Iden)o(ti\014ers)h(are)f(limited)i(to)e(thirt)o(y)l(,)g (not)f(six,)i(signi\014can)o(t)g(c)o(haracters.)143 1071 y FA(\017)23 b FC(MPI)15 b(Iden)o(ti\014ers)h(ma)o(y)f(con)o(tain)g (underscores)h(after)e(the)i(\014rst)e(c)o(haracter.)143 1171 y FA(\017)23 b FC(An)c(MPI)f(subroutine)i(with)f(a)f(c)o(hoice)i (argumen)o(t)e(ma)o(y)g(b)q(e)h(called)i(with)e(di\013eren)o(t)g (argumen)o(t)189 1227 y(t)o(yp)q(es.)32 b(An)20 b(example)g(is)g(sho)o (wn)f(in)h(Figure)g(2.1.)32 b(This)20 b(violates)g(the)f(letter)h(of)f (the)g(F)l(ortran)189 1284 y(standard,)14 b(but)h(suc)o(h)g(a)f (violation)i(is)f(common)f(practice.)21 b(The)15 b(alternativ)o(e)g(w)o (ould)g(b)q(e)g(to)f(ha)o(v)o(e)189 1340 y(a)h(separate)f(v)o(ersion)i (of)e Fv(MPI)p 697 1340 14 2 v 16 w(SEND)p FC(,)h(for)g(example,)h(for) e(eac)o(h)h(data)g(t)o(yp)q(e.)143 1439 y FA(\017)23 b FC(Although)12 b(not)f(required,)i(it)f(is)g(strongly)f(suggested)h (that)f(named)h(MPI)f(constan)o(ts)g(b)q(e)h(pro)o(vided)189 1496 y(in)18 b(an)g(include)i(\014le,)g(called)f Fo(mpif.h)p FC(.)27 b(On)19 b(systems)e(that)g(do)h(not)f(supp)q(ort)h(include)j (\014les,)e(the)189 1552 y(implemen)o(tation)d(should)g(sp)q(ecify)h (the)e(v)m(alues)h(of)f(named)h(constan)o(ts.)143 1651 y FA(\017)23 b FC(V)l(endors)e(are)f(encouraged)h(to)f(pro)o(vide)h(t)o (yp)q(e)g(declarations)g(in)h(the)f(mpif.h)g(\014le)h(on)e(F)l(ortran) 189 1708 y(systems)h(that)f(supp)q(ort)i(user-de\014ned)h(t)o(yp)q(es.) 39 b(One)22 b(should)h(de\014ne,)h(if)e(p)q(ossible,)i(the)e(t)o(yp)q (e)189 1764 y Fv(MPI)p 274 1764 V 15 w(ADDR)p FC(,)15 b(whic)o(h)g(is)f(an)h Fv(INTEGER)g FC(of)e(the)i(size)g(needed)g(to)f (hold)h(an)f(address)h(in)g(the)f(execu-)189 1821 y(tion)h(en)o (vironmen)o(t.)22 b(On)16 b(systems)f(where)g(t)o(yp)q(e)h (de\014nition)h(is)f(not)f(supp)q(orted,)h(it)g(is)g(up)g(to)f(the)189 1877 y(user)h(to)f(use)i(an)f Fv(INTEGER)g FC(of)g(the)g(righ)o(t)g (kind)h(to)e(represen)o(t)h(addresses)h(\(i.e.,)e Fo(INTEGER*4)g FC(on)189 1934 y(a)g(32)f(bit)i(mac)o(hine,)f Fo(INTEGER*8)f FC(on)i(a)e(64)h(bit)h(mac)o(hine,)f(etc.\).)75 2104 y Fp(2.4.3)49 b(C)16 b(Binding)i(Issues)75 2193 y FC(W)l(e)23 b(use)g(the)f(ANSI)i(C)e(declaration)h(format.)41 b(All)24 b(MPI)f(names)f(ha)o(v)o(e)g(an)h Fo(MPI)p 1548 2193 15 2 v 39 w FC(pre\014x,)i(de\014ned)75 2249 y(constan)o(ts)15 b(are)h(in)h(all)g(capital)g(letters,)f(and)g(de\014ned)i(t)o(yp)q(es)e (and)g(functions)h(ha)o(v)o(e)f(one)g(capital)h(letter)75 2306 y(after)d(the)i(pre\014x.)166 2364 y(Almost)24 b(all)h(C)e (functions)i(return)f(an)g(error)f(co)q(de.)47 b(The)24 b(successful)h(return)f(co)q(de)h(will)g(b)q(e)75 2420 y Fo(MPI)p 150 2420 V 17 w(SUCCESS)p FC(,)16 b(but)i(failure)h(return)f (co)q(des)g(are)f(implemen)o(tation)i(dep)q(enden)o(t.)29 b(A)18 b(few)g(C)f(functions)75 2476 y(will)i(return)f(v)m(alues,)h(so) e(that)g(they)g(can)h(b)q(e)g(implemen)o(ted)i(as)d(macros.)26 b(Logical)18 b(\015ags)g(are)f(in)o(tegers)75 2533 y(with)c(v)m(alues)h (1)e(and)h(0)g(meaning)g(\\true")f(and)h(\\false")g(resp)q(ectiv)o(ely) l(.)21 b(Choice)13 b(argumen)o(ts)f(are)h(p)q(oin)o(ters)75 2589 y(of)i(t)o(yp)q(e)g Fo(void*)p FC(.)166 2647 y(The)f(t)o(yp)q(e)f Fo(MPI)p 433 2647 V 17 w(Aint)g FC(is)h(de\014ned)h(to)d(b)q(e)i(an)g Fo(int)f FC(of)g(the)g(size)i(needed)g(to)d(hold)j(an)e(address)g(on)h (the)75 2704 y(target)g(arc)o(hitecture.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 11 19 11 18 bop 75 -100 a Fs(2.5.)34 b(PR)o(OCESSES)1381 b FC(11)75 45 y Ft(2.5)59 b(Pro)r(cesses)75 147 y FC(An)14 b(MPI)g(program)e(consists)i(of)f(autonomous)g(pro)q(cesses,)h (executing)h(their)f(o)o(wn)f(co)q(de,)h(in)h(an)f(MIMD)75 203 y(st)o(yle.)29 b(The)18 b(co)q(des)h(executed)g(b)o(y)f(eac)o(h)g (pro)q(cess)h(need)g(not)f(b)q(e)h(iden)o(tical.)30 b(The)19 b(pro)q(cesses)f(comm)o(u-)75 259 y(nicate)g(via)g(calls)g(to)f(MPI)g (comm)o(unication)h(primitiv)o(es.)28 b(T)o(ypically)l(,)19 b(eac)o(h)f(pro)q(cess)g(executes)g(in)g(its)75 316 y(o)o(wn)h(address) g(space,)h(although)f(shared-memory)g(implemen)o(tations)h(of)f(MPI)g (are)g(p)q(ossible.)33 b(This)75 372 y(do)q(cumen)o(t)20 b(sp)q(eci\014es)h(the)f(b)q(eha)o(vior)g(of)f(a)g(parallel)i(program)d (assuming)i(that)e(only)i(MPI)g(calls)g(are)75 429 y(used)c(for)e(comm) o(unication.)21 b(The)15 b(in)o(teraction)h(of)f(an)g(MPI)g(program)f (with)i(other)e(p)q(ossible)j(means)e(of)75 485 y(comm)o(unication)h (\(e.g.,)e(shared)h(memory\))f(is)i(not)f(sp)q(eci\014ed.)166 542 y(MPI)c(do)q(es)g(not)g(sp)q(ecify)h(the)f(execution)h(mo)q(del)g (for)e(eac)o(h)h(pro)q(cess.)19 b(A)11 b(pro)q(cess)g(can)h(b)q(e)f (sequen)o(tial,)75 598 y(or)f(can)h(b)q(e)h(m)o(ulti-threaded,)g(with)f (threads)g(p)q(ossibly)h(executing)g(concurren)o(tly)l(.)20 b(Care)10 b(has)g(b)q(een)i(tak)o(en)75 655 y(to)j(mak)o(e)f(MPI)h (\\thread-safe,")f(b)o(y)i(a)o(v)o(oiding)f(the)g(use)h(of)f(implicit)i (state.)166 711 y(MPI)e(do)q(es)h(not)f(sp)q(ecify)i(either)f(the)f(in) o(teraction)h(of)f(pro)q(cesses)h(with)g(signals,)g(in)g(a)f(UNIX)h(en) o(vi-)75 768 y(ronmen)o(t,)11 b(or)g(with)h(other)f(ev)o(en)o(ts)g (that)g(do)g(not)h(relate)f(to)g(MPI)g(comm)o(unication.)20 b(That)11 b(is,)h(signals)g(are)75 824 y(not)h(signi\014can)o(t)h(from) e(the)h(view)h(p)q(oin)o(t)g(of)e(MPI,)h(and)g(implemen)o(tors)h (should)g(attempt)e(to)h(implemen)o(t)75 880 y(MPI)h(so)g(that)f (signals)i(are)f(transparen)o(t:)19 b(an)14 b(MPI)g(call)h(susp)q (ended)h(b)o(y)e(a)g(signal)h(should)g(resume)g(and)75 937 y(complete)h(after)e(the)i(signal)g(is)f(handled.)166 993 y(MPI)j(do)q(es)g(not)f(pro)o(vide)h(mec)o(hanisms)g(to)f(sp)q (ecify)i(the)f(initial)i(allo)q(cation)f(of)e(pro)q(cesses)h(to)f(an)75 1050 y(MPI)j(computation)g(and)h(their)f(binding)j(to)c(ph)o(ysical)j (pro)q(cessors.)34 b(It)21 b(is)f(exp)q(ected)i(that)d(v)o(endors)75 1106 y(will)d(pro)o(vide)f(mec)o(hanisms)h(to)e(do)g(so)h(either)g(at)f (load)h(time)g(or)f(at)g(run)h(time.)20 b(Suc)o(h)15 b(mec)o(hanisms)g(will)75 1163 y(allo)o(w)d(the)f(sp)q(eci\014cation)i (of)e(the)g(initial)j(n)o(um)o(b)q(er)d(of)g(required)i(pro)q(cesses,)f (the)f(co)q(de)h(to)f(b)q(e)h(executed)g(b)o(y)75 1219 y(eac)o(h)g(initial)i(pro)q(cess,)f(and)f(the)g(allo)q(cation)h(of)f (pro)q(cesses)g(to)g(pro)q(cessors.)18 b(Also,)13 b(the)f(curren)o(t)g (prop)q(osal)75 1276 y(do)q(es)19 b(not)f(pro)o(vide)h(for)e(dynamic)j (creation)e(or)g(deletion)i(of)e(pro)q(cesses)g(during)i(program)d (execution)75 1332 y(\(the)f(total)f(n)o(um)o(b)q(er)h(of)f(pro)q (cesses)i(is)f(\014xed\),)g(although)g(it)g(is)g(in)o(tended)i(to)d(b)q (e)h(consisten)o(t)g(with)h(suc)o(h)75 1389 y(extensions.)26 b(Finally)l(,)19 b(the)f(curren)o(t)f(prop)q(osal)g(do)q(es)g(not)g(sp) q(ecify)i(a)d(naming)i(sc)o(heme)f(for)g(pro)q(cesses.)75 1445 y(W)l(e)c(prop)q(ose)h(alw)o(a)o(ys)e(to)h(iden)o(tify)h(pro)q (cesses)g(according)f(to)g(their)h(relativ)o(e)f(rank)g(in)h(a)f (group,)g(so)g(that,)75 1501 y(e\013ectiv)o(ely)l(,)j(pro)q(cesses)g (are)e(iden)o(ti\014ed)k(b)o(y)d(consecutiv)o(e)h(in)o(tegers)f(in)h (the)g(range)e Fo(0..groupsize-1)p FC(.)75 1642 y Ft(2.6)59 b(Erro)n(r)21 b(Handling)75 1744 y FC(MPI)11 b(pro)o(vides)h(the)g (user)f(with)h(reliable)i(message)d(transmission.)19 b(A)11 b(message)g(sen)o(t)g(is)h(alw)o(a)o(ys)f(receiv)o(ed)75 1800 y(correctly)l(,)17 b(and)g(the)g(user)g(do)q(es)g(not)f(need)i(to) e(c)o(hec)o(k)h(for)f(transmission)h(errors,)f(time-outs,)h(or)f(other) 75 1857 y(error)11 b(conditions.)20 b(In)13 b(other)f(w)o(ords,)f(MPI)h (do)q(es)g(not)g(pro)o(vide)g(mec)o(hanisms)h(for)e(dealing)j(with)e (failures)75 1913 y(in)17 b(the)f(comm)o(unication)h(system.)k(If)c (the)f(MPI)g(implemen)o(tation)h(is)f(built)i(on)e(an)g(unreliable)i (under-)75 1970 y(lying)h(mec)o(hanism,)f(then)g(it)f(is)h(the)g(job)f (of)g(the)g(implemen)o(tor)i(of)e(the)g(MPI)h(subsystem)f(to)g (insulate)75 2026 y(the)c(user)h(from)e(this)i(unreliabilit)o(y)l(,)i (or)d(to)f(re\015ect)i(unreco)o(v)o(erable)f(errors)g(as)f(global)i (program)e(failures.)75 2083 y(Similarly)l(,)i(MPI)e(itself)h(pro)o (vides)f(no)g(mec)o(hanisms)h(for)e(handling)i(no)q(de)g(failures.)20 b(\(A)11 b(mec)o(hanism)i(that)75 2139 y(allo)o(ws)i(the)h(user)f(to)g (restrict)g(the)g(scop)q(e)h(of)e(an)i(unreco)o(v)o(erable)f(error)g (is)h(presen)o(ted)f(in)h(section)g(7.2.\))166 2195 y(Of)g(course,)h (MPI)f(programs)f(ma)o(y)h(still)i(b)q(e)f(erroneous.)23 b(A)16 b Fx(program)i(error)d FC(can)i(o)q(ccur)f(when)75 2252 y(an)g(MPI)g(call)g(is)h(called)g(with)f(an)g(incorrect)g(argumen) o(t)f(\(non-existing)i(destination)g(in)g(a)e(send)i(op)q(er-)75 2308 y(ation,)g(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i(op)q (eration,)f(etc.\))25 b(This)17 b(t)o(yp)q(e)g(of)f(error)g(w)o(ould)h (o)q(ccur)h(in)f(an)o(y)75 2365 y(implemen)o(tation.)31 b(In)19 b(addition,)h(a)f Fx(resource)h(error)d FC(ma)o(y)h(o)q(ccur)h (when)g(a)f(program)f(exceeds)j(the)75 2421 y(amoun)o(t)d(of)f(a)o(v)m (ailable)j(system)e(resources)h(\(n)o(um)o(b)q(er)f(of)g(p)q(ending)i (messages,)e(system)g(bu\013ers,)g(etc.\).)75 2478 y(The)h(o)q (ccurrence)h(of)f(this)g(t)o(yp)q(e)g(of)g(error)f(dep)q(ends)j(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(in)h(the)75 2534 y(system)13 b(and)h(the)g(resource)f(allo)q(cation)i(mec)o(hanism) f(used;)h(this)f(ma)o(y)f(di\013er)h(from)f(system)g(to)g(system.)75 2591 y(A)j(high)h(qualit)o(y)g(implemen)o(tation)g(will)h(pro)o(vide)e (generous)h(limits)g(on)f(some)g(of)f(the)i(resources)f(so)f(as)75 2647 y(to)h(alleviate)i(the)f(p)q(ortabilit)o(y)h(problem)f(this)h (represen)o(ts.)24 b(A)17 b(program)f(is)h Fx(safe)g FC(if)g(it)g(is)h(not)e(sub)s(ject)75 2704 y(to)f(resource)g(errors,)f Fx(unsafe)p FC(,)h(otherwise.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 12 20 12 19 bop 75 -100 a FC(12)692 b Fs(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)166 45 y FC(All)k(MPI)f(calls)g (return)g(a)g(co)q(de)g(that)f(indicates)i(successful)g(completion)g (of)e(the)h(op)q(eration,)g(or)75 102 y(the)g(error)f(condition)i(that) f(o)q(ccurred,)h(otherwise.)28 b(Ho)o(w)o(ev)o(er,)17 b(MPI)h(do)q(es)g(not)g(require)h(the)f(user)g(to)75 158 y(c)o(hec)o(k)e(for)e(error)h(v)m(alues.)22 b(It)15 b(is)h(recommended)h(that)d(an)i(implemen)o(tation)g(pro)o(vide)g(an)g (en)o(vironmen)o(t)75 214 y(for)f(MPI)h(execution)g(that)f(can)h(b)q(e) g(set)g(so)f(that)g(an)o(y)g(error)g(encoun)o(tered)h(in)h(an)e(MPI)h (call)g(will)i(cause)75 271 y(a)d(program)g(to)g(fail)i(with)f(a)f (suitable)i(error)e(message)g(returned)h(to)f(the)h(user.)22 b(The)16 b(en)o(vironmen)o(t)g(for)75 327 y(an)f(MPI)g(execution)h(can) f(also)f(b)q(e)i(set)f(so)f(that)g(\(non-fatal\))g(errors)h(do)f(not)h (cause)g(a)g(program)e(to)i(fail,)75 384 y(and)j(the)f(user)g(can)h (pro)o(vide)g(his)g(or)f(her)g(o)o(wn)g(error)g(reco)o(v)o(ery)f(co)q (de.)27 b(A)18 b(mec)o(hanism)g(that)e(ac)o(hiev)o(es)75 440 y(this)g(in)g(a)f(POSIX)h(en)o(vironmen)o(t)f(is)h(describ)q(ed)h (in)f(section)g(7.2.)166 500 y(MPI)d(calls)h(ma)o(y)f(initiate)h(op)q (erations)g(that)e(con)o(tin)o(ue)i(async)o(hronously)g(after)e(the)i (call)g(returned.)75 557 y(Th)o(us,)22 b(the)g(op)q(eration)f(ma)o(y)g (return)g(with)h(a)f(co)q(de)h(indicating)h(successful)g(completion,)g (y)o(et)e(later)75 613 y(cause)15 b(an)g(error)g(exception)h(to)e(b)q (e)i(raised.)k(If)15 b(there)g(is)h(a)f(subsequen)o(t)g(call)h(that)f (relates)g(to)f(the)h(same)75 669 y(op)q(eration)f(\(e.g.,)f(a)g(call)i (that)e(v)o(eri\014es)i(that)e(an)h(async)o(hronous)f(op)q(eration)h (has)g(completed\))h(then)f(the)75 726 y(error)g(argumen)o(t)g(asso)q (ciated)h(with)g(this)g(call)g(will)i(b)q(e)e(used)g(to)f(indicate)i (the)f(nature)f(of)h(the)f(error.)19 b(In)75 782 y(a)14 b(few)h(cases,)f(the)g(error)g(ma)o(y)g(o)q(ccur)h(after)f(all)h(calls) h(that)d(relate)i(to)f(the)h(op)q(eration)f(ha)o(v)o(e)g(completed,)75 839 y(so)e(that)g(no)g(error)g(v)m(alue)i(can)e(b)q(e)h(used)g(to)f (indicate)i(the)f(nature)f(of)g(the)h(error)e(\(e.g.,)h(an)g(error)g (in)h(a)f(send)75 895 y(with)j(the)g(ready)f(mo)q(de\).)20 b(Suc)o(h)15 b(an)g(error)f(m)o(ust)g(b)q(e)i(treated)e(as)g(fatal,)g (since)i(information)f(cannot)f(b)q(e)75 952 y(returned)i(for)e(the)h (user)h(to)e(reco)o(v)o(er)h(from)f(it.)166 1011 y(The)21 b(b)q(eha)o(vior)g(of)g(erroneous)f(programs)g(is)h(not)g(de\014ned)h (b)o(y)e(this)i(do)q(cumen)o(t.)37 b(The)21 b(desired)75 1068 y(b)q(eha)o(vior)14 b(is)g(that)e(a)i(relev)m(an)o(t)g(error)e(co) q(de)i(b)q(e)g(returned,)g(if)g(no)f(acceptable)i(b)q(eha)o(vior)e(can) h(b)q(e)g(de\014ned.)75 1124 y(Ho)o(w)o(ev)o(er,)23 b(e\016ciency)g (considerations)g(ma)o(y)f(dictate)g(that)g(the)g(system)g(do)q(es)g (not)g(detect)h(certain)75 1181 y(t)o(yp)q(es)c(of)f(errors.)29 b(Also,)19 b(implemen)o(tations)h(ma)o(y)d(go)h(b)q(ey)o(ond)i(this)e (do)q(cumen)o(t)h(in)h(supp)q(orting)f(in)g(a)75 1237 y(meanningful)e(manners)e(co)q(des)h(that)e(are)h(de\014ned)i(here)e (to)g(b)q(e)h(erroneous.)166 1297 y(MPI,)10 b(lik)o(e)h(an)o(y)f(other) g(similar)h(message-passing)f(library)l(,)i(allo)o(ws)e(users)g(to)g (write)g(non-deterministic)75 1354 y(programs,)16 b(where)h(race)f (conditions)i(exist)g(and)f(di\013eren)o(t)g(timing)g(of)f(comm)o (unication)i(ma)o(y)e(lead)i(to)75 1410 y(di\013eren)o(t)g(results.)26 b(There)18 b(is)g(no)f(requiremen)o(t)h(that)f(rep)q(eated)h (executions)g(of)f(the)g(same)g(suc)o(h)h(pro-)75 1466 y(gram)e(will)i(yield)g(the)f(same)f(result,)h(ev)o(en)g(when)g(rep)q (eated)g(with)g(the)g(same)f(ph)o(ysical)i(con\014guration)75 1523 y(|)d(neither)g(user)f(nor)g(MPI)g(can)g(b)q(e)h(exp)q(ected)g(to) f(ha)o(v)o(e)g(full)h(con)o(trol)f(of)g(the)g(execution)h(en)o (vironmen)o(t.)75 1579 y(Ho)o(w)o(ev)o(er,)e(rep)q(eatabilit)o(y)j(is)f (a)f(desirable)i(qualit)o(y-of-implemen)o(tation)g(goal.)j(In)c (particular,)g(it)g(is)g(ex-)75 1636 y(p)q(ected)i(that)f(MPI)h (implemen)o(tations)g(will)h(a)o(v)o(oid)f(in)o(tro)q(ducing)g (additional)h(non-determinism.)26 b(F)l(or)75 1692 y(example,)19 b(while)g(MPI)f(can)g(tak)o(e)f(adv)m(an)o(tage)g(of)g(asso)q(ciativit) o(y)h(and)g(comm)o(utativit)o(y)f(in)i(reordering)75 1749 y(the)f(execution)g(of)f(a)h(global)g(reduce)g(op)q(eration,)g(it) g(is)g(desired)h(\(but)e(not)g(required\))i(that)e(the)g(same)75 1805 y(op)q(erations)e(order)g(b)q(e)h(used)g(in)g(rep)q(eated)g (executions.)75 1967 y Ft(2.7)59 b(Implementation)17 b(issues)75 2075 y FC(There)c(are)f(a)g(n)o(um)o(b)q(er)h(of)f(areas)f (where)i(an)f(MPI)h(implemen)o(tation)g(ma)o(y)f(in)o(teract)g(with)h (the)g(op)q(erating)75 2132 y(en)o(vironmen)o(t)k(and)f(system.)23 b(While)18 b(MPI)e(do)q(es)h(not)f(mandate)g(that)f(an)o(y)h(services)h (\(suc)o(h)g(as)f(I/O)g(or)75 2188 y(signal)h(handling\))g(b)q(e)f(pro) o(vided,)g(it)g(do)q(es)g(strongly)g(suggest)f(the)h(b)q(eha)o(vior)g (to)f(b)q(e)h(pro)o(vided)h(if)f(those)75 2245 y(services)g(are)g(a)o (v)m(ailable.)23 b(This)16 b(is)g(an)f(imp)q(ortan)o(t)h(p)q(oin)o(t)g (in)g(ac)o(hieving)h(p)q(ortabilit)o(y)g(across)d(platforms)75 2301 y(that)g(pro)o(vide)i(the)g(same)e(set)h(of)g(services.)75 2442 y Fp(2.7.1)49 b(Indep)q(endence)15 b(of)i(basic)f(runtime)g (routines)75 2534 y FC(MPI)d(programs)e(require)j(that)e(library)i (routines)f(that)f(are)h(part)f(of)g(the)h(basic)h(language)f(en)o (vironmen)o(t)75 2591 y(\(suc)o(h)i(as)g Fo(date)f FC(and)i Fo(write)e FC(in)i(F)l(ortran)e(and)h Fo(printf)f FC(and)h Fo(malloc)g FC(in)h(ANSI)f(C\))g(and)g(are)g(executed)75 2647 y(after)g Fo(MPI)p 258 2647 15 2 v 17 w(INIT)g FC(and)i(b)q(efore) f Fo(MPI)p 685 2647 V 17 w(FINALIZE)f FC(op)q(erate)g(indep)q(enden)o (tly)k(and)e(that)e(their)h Fr(c)n(ompletion)75 2704 y FC(is)g(indep)q(enden)o(t)h(of)e(the)g(action)h(of)e(other)h(pro)q (cesses)h(in)g(an)f(MPI)g(program.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 13 21 13 20 bop 75 -100 a Fs(2.7.)34 b(IMPLEMENT)l(A)l(TION)16 b(ISSUES)1045 b FC(13)166 45 y(Note)16 b(that)g(this)h(in)g(no)g(w)o(a) o(y)e(prev)o(en)o(ts)i(the)f(creation)h(of)f(library)i(routines)f(that) f(pro)o(vide)h(paral-)75 102 y(lel)j(services)g(whose)f(op)q(eration)g (is)g(collectiv)o(e.)33 b(Ho)o(w)o(ev)o(er,)18 b(the)h(follo)o(wing)g (program)f(is)i(exp)q(ected)g(to)75 158 y(complete)c(in)g(an)f(ANSI)h (C)f(en)o(vironmen)o(t)h(regardless)f(of)g(the)g(size)h(of)f Fo(MPI)p 1371 158 15 2 v 17 w(COMM)p 1484 158 V 16 w(WORLD)p FC(:)75 252 y Fo(int)23 b(rank;)75 308 y(MPI_Init\()g(argc,)g(argv)g (\);)75 365 y(MPI_Comm_rank\()f(MPI_COMM_WORLD,)g(&rank)h(\);)75 421 y(if)h(\(rank)f(==)g(0\))h(printf\()f("Starting)f(program\\n")h (\);)75 478 y(MPI_Finalize\(\);)75 571 y FC(The)15 b(corresp)q(onding)i (F)l(ortran)d(77)g(program)g(is)i(also)f(exp)q(ected)i(to)d(complete.) 166 628 y(An)k(example)h(of)f(what)f(is)i Fr(not)j FC(required)d(is)g (an)o(y)f(particular)h(ordering)f(of)g(the)g(action)g(of)g(these)75 684 y(routines)h(when)h(called)g(b)o(y)f(sev)o(eral)h(tasks.)30 b(F)l(or)18 b(example,)j(MPI)e(mak)o(es)f(neither)i(requiremen)o(ts)f (or)75 741 y(recommendations)c(for)g(the)g(output)g(from)75 835 y Fo(MPI_Comm_rank\()22 b(MPI_COMM_WORLD,)g(&rank)h(\);)75 891 y(printf\()g("Output)g(from)g(task)g(rank)h(\045d\\n",)f(rank)g (\);)166 985 y FC(In)18 b(addition,)h(calls)f(that)f(fail)h(b)q(ecause) g(of)f(resource)h(exhaustion)g(or)f(other)g(error)g(are)g(not)g(con-)75 1041 y(sidered)h(a)f(violation)h(of)e(the)h(requiremen)o(ts)h(here)f (\(ho)o(w)o(ev)o(er,)f(they)h(are)g(required)h(to)e(complete,)i(just)75 1098 y(not)d(to)f(complete)i(successfully\).)75 1220 y Fp(2.7.2)49 b(Interaction)16 b(with)g(signals)i(in)f(POSIX)75 1305 y FC(The)e(in)o(ten)o(t)g(of)f(MPI)h(is)g(to)f(b)q(e)h(thread)g (and)f(signal)i(safe.)j(This)d(has)e(a)g(n)o(um)o(b)q(er)h(of)g(subtle) g(e\013ects.)20 b(F)l(or)75 1362 y(example,)g(on)e(Unix)h(systems,)g(a) f(catc)o(hable)h(signal)g(suc)o(h)g(as)f(SIGALRM)h(\(an)f(alarm)h (signal\))g(m)o(ust)75 1418 y(not)f(cause)g(an)h(MPI)f(routine)g(to)g (b)q(eha)o(v)o(e)h(di\013eren)o(tly)g(than)f(it)g(w)o(ould)h(ha)o(v)o (e)f(in)h(the)f(absence)h(of)f(the)75 1475 y(signal.)37 b(Of)21 b(course,)h(if)f(the)g(signal)h(handler)f(issues)h(MPI)f(calls) g(or)g(c)o(hanges)f(the)h(en)o(vironmen)o(t)g(in)75 1531 y(whic)o(h)c(the)g(MPI)g(routine)g(is)g(op)q(erating)f(\(for)g (example,)h(consuming)h(all)f(a)o(v)m(ailable)h(memory)e(space\),)75 1588 y(the)e(MPI)f(routine)h(should)h(b)q(eha)o(v)o(e)f(as)f (appropriate)h(for)f(that)g(situation)h(\(in)g(particular,)g(in)g(this) g(case,)75 1644 y(the)h(b)q(eha)o(vior)h(should)g(b)q(e)g(the)f(same)g (as)g(for)g(a)g(m)o(ultithreaded)h(MPI)f(implemen)o(tation\).)166 1701 y(A)20 b(second)g(e\013ect)g(is)g(that)g(a)f(signal)i(handler)g (that)e(p)q(erforms)h(MPI)g(calls)h(m)o(ust)e(not)h(in)o(terfere)75 1757 y(with)c(the)h(op)q(eration)f(of)g(MPI.)f(F)l(or)h(example,)g(an)h (MPI)f(receiv)o(e)h(of)e(an)o(y)h(t)o(yp)q(e)g(that)g(o)q(ccurs)g (within)h(a)75 1813 y(signal)c(handler)f(m)o(ust)f(not)h(cause)g (erroneous)f(b)q(eha)o(vior)i(b)o(y)e(the)h(MPI)g(implemen)o(tation.)20 b(Note)11 b(that)g(an)75 1870 y(implemen)o(tation)16 b(is)g(p)q(ermitted)g(to)e(prohibit)i(the)f(use)h(of)e(MPI)h(calls)h (from)e(within)j(a)d(signal)i(handler,)75 1926 y(and)f(is)h(not)f (required)h(to)f(detect)g(suc)o(h)h(use.)166 1983 y(It)d(is)h(highly)g (desirable)h(that)d(MPI)h(not)g(use)g Fo(SIGALRM)p FC(,)f Fo(SIGFPE)p FC(,)g(or)g Fo(SIGIO)p FC(.)g(An)i(implemen)o(tation)75 2039 y(is)g Fr(r)n(e)n(quir)n(e)n(d)k FC(to)c(clearly)h(do)q(cumen)o(t) f(all)h(of)f(the)g(signals)h(that)e(the)h(MPI)g(implemen)o(tation)h (uses;)f(a)g(go)q(o)q(d)75 2096 y(place)i(for)f(this)g(information)h (is)f(a)g(Unix)h(`)p Fo(man)p FC(')e(page)h(on)h Fo(MPI)p FC(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 14 22 14 21 bop 75 391 a Fu(Chapter)34 b(3)75 633 y Fy(P)m(oin)m(t)41 b(to)f(P)m(oin)m(t)h(Comm)m(unication)75 908 y Ft(3.1)59 b(Intro)r(duction)75 1076 y FC(Sending)13 b(and)f(receiving)h(of)e (messages)f(b)o(y)i(pro)q(cesses)g(is)g(the)f(basic)h(MPI)g(comm)o (unication)g(mec)o(hanism.)75 1133 y(All)20 b(other)f(comm)o(unication) h(functions)g(can)f(b)q(e)h(de\014ned)h(and)e(implemen)o(ted)i(on)e (top)g(of)g(this)g(basic)75 1189 y(comm)o(unication)e(la)o(y)o(er,)f (using)h(few)f(auxiliary)i(data)e(managemen)o(t)f(routines,)i(and)g (using)g(the)f(group)75 1246 y(and)f(comm)o(unicator)g(manipulation)i (functions.)166 1336 y(The)i(basic)g(p)q(oin)o(t)g(to)f(p)q(oin)o(t)h (comm)o(unication)g(op)q(erations)g(are)f Fx(send)g FC(and)h Fx(receiv)o(e)p FC(.)29 b(A)18 b Fx(send)75 1393 y FC(op)q(eration)13 b(creates)f(and)h(sends)g(a)g(message.)18 b(The)13 b(op)q(eration)g(sp) q(eci\014es)i(a)d Fx(send)i(bu\013er)e FC(in)i(the)e(sender)75 1449 y(memory)f(from)g(whic)o(h)h(the)g(message)f(data)g(is)h(tak)o (en.)18 b(In)12 b(addition,)h(the)f(send)g(op)q(eration)g(asso)q (ciates)f(an)75 1506 y Fx(en)o(v)o(elop)q(e)16 b FC(with)g(the)g (message.)22 b(This)16 b(en)o(v)o(elop)q(e)h(sp)q(eci\014es)h(the)e (message)g(destination)h(and)f(con)o(tains)75 1562 y(distinguishing)h (information)d(that)f(can)i(b)q(e)g(used)f(b)o(y)g(the)g Fx(receiv)o(e)g FC(op)q(eration)g(to)g(select)h(a)e(particular)75 1619 y(message.)166 1710 y(A)22 b Fx(receiv)o(e)f FC(op)q(eration)h (consumes)g(a)f(message.)39 b(The)22 b(message)f(to)g(b)q(e)i(receiv)o (ed)f(is)h(selected)75 1766 y(according)13 b(to)f(the)g(v)m(alue)i(on)e (its)g(en)o(v)o(elop)q(e,)i(and)f(the)f(message)g(data)g(is)h(put)f(in) o(to)g(the)h Fx(receiv)o(e)g(bu\013er)p FC(.)166 1857 y(The)f(next)g(sections)h(describ)q(e)g(the)g(basic)f(\(blo)q(c)o (king\))h(send)g(and)f(receiv)o(e)h(op)q(erations.)19 b(W)l(e)12 b(discuss)75 1913 y(send,)21 b(receiv)o(e,)g(basic)f(comm)o (unication)g(seman)o(tics,)h(t)o(yp)q(e)e(matc)o(hing)h(requiremen)o (ts,)h(t)o(yp)q(e)e(con)o(v)o(er-)75 1970 y(sion)14 b(in)h (heterogeneous)f(en)o(vironmen)o(ts,)f(and)h(more)g(general)g(comm)o (unication)g(mo)q(des.)20 b(Non)o(blo)q(c)o(king)75 2026 y(comm)o(unication)c(is)h(addressed)f(next,)f(follo)o(w)o(ed)h(b)o(y)g (c)o(hannel-lik)o(e)i(constructs)e(and)f(send-receiv)o(e)j(op-)75 2083 y(erations.)i(W)l(e)15 b(then)g(consider)h(general)f(datat)o(yp)q (es)f(that)g(allo)o(w)h(one)g(to)g(transfer)f(heterogeneous)h(and)75 2139 y(noncon)o(tiguous)f(data,)f(and)h(conclude)h(with)f(a)g (description)h(of)e(an)h(implemen)o(tation)h(of)e(MPI)g(p)q(oin)o(t)i (to)75 2196 y(p)q(oin)o(t)h(comm)o(unication)g(using)g(a)e(small)i(n)o (um)o(b)q(er)g(of)f(primitiv)o(es.)75 2535 y Ft(3.2)59 b(Basic)19 b(send)h(op)r(eration)75 2704 y FC(The)15 b(syn)o(tax)g(of)g(the)g(simplest)h(send)g(op)q(eration)f(is)h(giv)o (en)g(b)q(elo)o(w.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 15 23 15 22 bop 75 -100 a Fs(3.2.)34 b(BASIC)16 b(SEND)f(OPERA)l(TION)1075 b FC(15)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SEND\(buf,)16 b(count,)g(datat)o(yp)q(e,)g(dest,)g(tag,)f(comm\))117 126 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 209 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)g(in)g(send)h(bu\013er)h (\(nonnegativ)o(e)e(in)o(te-)905 265 y(ger\))117 348 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 431 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o (teger\))117 514 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 596 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))75 725 y Fo(int)23 b(MPI)p 245 725 15 2 v 17 w(Send\(void*)g(buf,)g(int)g(count,)h(MPI)p 979 725 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 781 y(int)h(tag,)f(MPI)p 683 781 V 17 w(Comm)g(comm\))75 871 y(MPI)p 150 871 V 17 w(SEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)h(COMM,)f(IERROR\))170 928 y()g(BUF\(*\))170 984 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 1158 y Fp(3.2.1)49 b(Message)16 b(data)75 1251 y FC(The)e(send)g (bu\013er)g(sp)q(eci\014ed)i(b)o(y)e(the)f Fv(MPI)p 803 1251 14 2 v 16 w(SEND)h FC(op)q(eration)g(consists)g(of)f Fv(count)i FC(successiv)o(e)g(en)o(tries)f(of)75 1308 y(the)f(t)o(yp)q(e)h(indicated)h(b)o(y)e Fv(datat)o(yp)q(e)p FC(,)i(starting)e(with)h(the)f(en)o(try)g(at)g(address)h Fv(buf)p FC(.)20 b(Note)13 b(that)g(w)o(e)g(sp)q(ecify)75 1364 y(the)19 b(message)f(length)i(in)g(terms)e(of)g(n)o(um)o(b)q(er)h (of)g Fr(elements)p FC(,)f(not)g(n)o(um)o(b)q(er)i(of)e Fr(bytes)p FC(.)30 b(The)19 b(former)f(is)75 1421 y(mac)o(hine)e(indep) q(enden)o(t)i(and)d(closer)h(to)e(the)h(application)i(lev)o(el.)166 1481 y(The)g(data)f(part)g(of)g(the)g(message)h(consists)f(of)h(a)f (sequence)i(of)e Fv(count)i FC(v)m(alues,)f(eac)o(h)g(of)f(the)h(t)o (yp)q(e)75 1537 y(indicated)j(b)o(y)e Fv(datat)o(yp)q(e)p FC(.)31 b Fv(count)20 b FC(ma)o(y)d(b)q(e)i(zero,)g(in)g(whic)o(h)g (case)g(the)f(data)g(part)f(of)h(the)h(message)e(is)75 1594 y(empt)o(y)l(.)i(The)13 b(basic)h(datat)o(yp)q(es)e(that)h(can)g (b)q(e)h(sp)q(eci\014ed)h(for)d(message)h(data)f(v)m(alues)i(corresp)q (ond)g(to)e(the)75 1650 y(basic)j(datat)o(yp)q(es)f(of)g(the)h(host)f (language.)19 b(The)c(p)q(ossible)h(v)m(alues)g(of)e(this)h(argumen)o (t)f(for)f(F)l(ortran)h(and)75 1707 y(the)h(corresp)q(onding)h(F)l (ortran)e(t)o(yp)q(es)i(are)f(listed)h(b)q(elo)o(w)p 472 1892 1006 2 v 471 1948 2 57 v 497 1931 a(MPI)f(datat)o(yp)q(e)p 1046 1948 V 294 w(F)l(ortran)f(datat)o(yp)q(e)p 1477 1948 V 472 1950 1006 2 v 471 2006 2 57 v 497 1989 a Fv(MPI)p 582 1989 14 2 v 16 w(INTEGER)p 1046 2006 2 57 v 287 w Fo(INTEGER)p 1477 2006 V 471 2063 V 497 2046 a Fv(MPI)p 582 2046 14 2 v 16 w(REAL)p 1046 2063 2 57 v 365 w Fo(REAL)p 1477 2063 V 471 2119 V 497 2102 a Fv(MPI)p 582 2102 14 2 v 16 w(DOUBLE)p 777 2102 V 17 w(PRECISION)p 1046 2119 2 57 v 50 w Fo(DOUBLE)23 b(PRECISION)p 1477 2119 V 471 2176 V 497 2159 a Fv(MPI)p 582 2159 14 2 v 16 w(COMPLEX)p 1046 2176 2 57 v 263 w Fo(COMPLEX)p 1477 2176 V 471 2232 V 497 2215 a Fv(MPI)p 582 2215 14 2 v 16 w(LOGICAL)p 1046 2232 2 57 v 291 w Fo(LOGICAL)p 1477 2232 V 471 2289 V 497 2272 a Fv(MPI)p 582 2272 14 2 v 16 w(CHARA)o(CTER)p 1046 2289 2 57 v 211 w Fo(CHARACTER\(1\))p 1477 2289 V 471 2345 V 497 2328 a Fv(MPI)p 582 2328 14 2 v 16 w(BYTE)p 1046 2345 2 57 v 1477 2345 V 472 2347 1006 2 v 75 2549 a FC(The)13 b(p)q(ossible)i(v)m(alues)g(for)d(this)i(argumen)o(t)f(for) f(C)h(and)h(the)f(corresp)q(onding)h(C)f(t)o(yp)q(es)g(are)g(listed)i (b)q(elo)o(w.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 16 24 16 23 bop 75 -100 a FC(16)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)p 466 5 1019 2 v 465 62 2 57 v 491 45 a FC(MPI)f(datat)o(yp)q(e)p 1004 62 V 259 w(C)g(datat)o(yp)q(e)p 1483 62 V 466 63 1019 2 v 465 120 2 57 v 491 103 a Fv(MPI)p 576 103 14 2 v 16 w(CHAR)p 1004 120 2 57 v 321 w Fo(signed)23 b(char)p 1483 120 V 465 176 V 491 159 a Fv(MPI)p 576 159 14 2 v 16 w(SHORT)p 1004 176 2 57 v 291 w Fo(signed)g(short)p 1483 176 V 465 233 V 491 216 a Fv(MPI)p 576 216 14 2 v 16 w(INT)p 1004 233 2 57 v 365 w Fo(signed)g(int)p 1483 233 V 465 289 V 491 272 a Fv(MPI)p 576 272 14 2 v 16 w(LONG)p 1004 289 2 57 v 321 w Fo(signed)g(long)g(int)p 1483 289 V 465 346 V 491 329 a Fv(MPI)p 576 329 14 2 v 16 w(UNSIGNED)p 815 329 V 17 w(CHAR)p 1004 346 2 57 v 81 w Fo(unsigned)f(char)p 1483 346 V 465 402 V 491 385 a Fv(MPI)p 576 385 14 2 v 16 w(UNSIGNED)p 815 385 V 17 w(SHORT)p 1004 402 2 57 v 51 w Fo(unsigned)g(short)i(int)p 1483 402 V 465 459 V 491 442 a Fv(MPI)p 576 442 14 2 v 16 w(UNSIGNED)p 1004 459 2 57 v 218 w Fo(unsigned)e(int)p 1483 459 V 465 515 V 491 498 a Fv(MPI)p 576 498 14 2 v 16 w(UNSIGNED)p 815 498 V 17 w(LONG)p 1004 515 2 57 v 81 w Fo(unsigned)g(long)i(int)p 1483 515 V 465 571 V 491 555 a Fv(MPI)p 576 555 14 2 v 16 w(FLO)o(A)l(T)p 1004 571 2 57 v 301 w Fo(float)p 1483 571 V 465 628 V 491 611 a Fv(MPI)p 576 611 14 2 v 16 w(DOUBLE)p 1004 628 2 57 v 262 w Fo(double)p 1483 628 V 465 684 V 491 667 a Fv(MPI)p 576 667 14 2 v 16 w(LONG)p 712 667 V 16 w(DOUBLE)p 1004 684 2 57 v 126 w Fo(long)f(double)p 1483 684 V 465 741 V 491 724 a Fv(MPI)p 576 724 14 2 v 16 w(BYTE)p 1004 741 2 57 v 1483 741 V 466 742 1019 2 v 166 918 a FC(The)16 b(datat)o(yp)q(e)f Fv(MPI)p 536 918 14 2 v 15 w(BYTE)i FC(do)q(es)e(not)g(corresp)q(ond)h(to)f(a)g(F)l (ortran)g(or)g(C)g(datat)o(yp)q(e.)20 b(A)c(v)m(alue)g(of)75 975 y(t)o(yp)q(e)i Fv(MPI)p 265 975 V 15 w(BYTE)h FC(consists)f(of)f(8) g(binary)h(digits.)28 b(A)18 b(b)o(yte)f(is)h(unin)o(terpreted)h(and)f (is)g(di\013eren)o(t)g(from)75 1031 y(a)f(c)o(haracter.)25 b(Di\013eren)o(t)17 b(mac)o(hines)g(ma)o(y)g(ha)o(v)o(e)g(di\013eren)o (t)g(represen)o(tations)g(for)g(c)o(haracters,)f(or)h(ma)o(y)75 1088 y(use)g(more)g(than)g(one)g(b)o(yte)g(to)g(represen)o(t)g(c)o (haracters.)25 b(On)18 b(the)f(other)g(hand,)g(a)g(b)o(yte)g(has)g(the) g(same)75 1144 y(binary)f(v)m(alue)g(on)f(all)h(mac)o(hines.)166 1202 y(MPI)11 b(supp)q(orts)g(the)f(basic)i(datat)o(yp)q(es)e(of)g(F)l (ortran)g(77)g(and)h(ANSI)h(C.)e(Additional)j(MPI)e(datat)o(yp)q(es)75 1258 y(should)32 b(b)q(e)g(pro)o(vided)f(if)h(the)f(host)f(language)h (has)g(additional)i(data)d(t)o(yp)q(es:)51 b(F)l(or)30 b(example,)75 1315 y Fv(MPI)p 160 1315 V 16 w(LONG)p 296 1315 V 17 w(LONG)p 433 1315 V 17 w(INT)p FC(,)13 b(for)h(64)g(bit)h(in)o(tegers)g(in)g(C;)f Fv(MPI)p 1113 1315 V 16 w(DOUBLE)p 1308 1315 V 16 w(COMPLEX)h FC(for)f(double)i (preci-)75 1371 y(sion)h(complex)g(in)g(F)l(ortran;)e Fv(MPI)p 667 1371 V 16 w(REAL2)p FC(,)h Fv(MPI)p 928 1371 V 16 w(REAL4)g FC(and)h Fv(MPI)p 1266 1371 V 15 w(REAL8)g FC(for)e(2,4)g(or)h(8)g(b)o(yte)g(reals,)75 1427 y(in)g(F)l(ortran;)e Fv(MPI)p 391 1427 V 15 w(INTEGER1)i(MPI)p 716 1427 V 16 w(INTEGER2)f FC(and)g Fv(MPI)p 1129 1427 V 16 w(INTEGER4)h FC(for)e(1,)h(2)g(or)f(4)h(b)o(yte)g(in)o(tegers)75 1484 y(in)h(F)l(ortran;)e(etc.)75 1612 y Fp(3.2.2)49 b(Message)16 b(envelop)q(e)75 1700 y FC(In)g(addition)g(to)e(the)h (data)g(part,)f(messages)g(con)o(tain)h(information)h(that)e(can)h(b)q (e)h(used)f(to)g(distinguish)75 1757 y(messages)h(and)g(selectiv)o(ely) i(receiv)o(e)f(them.)23 b(This)17 b(information)f(is)h(con)o(tained)g (in)g(a)f(\014xed)h(n)o(um)o(b)q(er)f(of)75 1813 y(\014xed-format)f (\014elds,)h(whic)o(h)g(w)o(e)f(collectiv)o(ely)i(call)g(the)e Fx(message)i(en)o(v)o(elop)q(e)p FC(.)i(These)d(\014elds)g(are)75 1912 y Fx(source)75 2010 y(destination)75 2109 y(tag)75 2208 y(con)o(text)166 2306 y FC(The)c(message)g(source)g(is)h (implicitly)i(determined)f(b)o(y)e(the)g(iden)o(tit)o(y)h(of)f(the)g (message)g(sender.)20 b(The)75 2362 y(other)15 b(\014elds)h(are)f(sp)q (eci\014ed)i(b)o(y)f(argumen)o(ts)e(in)i(the)f(send)h(op)q(eration.)166 2420 y(The)h(in)o(teger-v)m(alued)h(message)e(tag)f(is)i(sp)q (eci\014ed)i(b)o(y)d(the)g Fv(tag)h FC(argumen)o(t.)22 b(This)17 b(in)o(teger)g(can)f(b)q(e)75 2477 y(used)c(b)o(y)g(the)f (program)g(to)g(distinguish)i(di\013eren)o(t)f(t)o(yp)q(es)g(of)f (messages.)18 b(The)12 b(range)f(of)g(v)m(alid)i(tag)e(v)m(alues)75 2533 y(is)18 b(implemen)o(tation)h(dep)q(enden)o(t)g(and)f(can)f(b)q(e) h(queried)h(using)f(the)g Fv(MPI)p 1362 2533 V 16 w(GET)p 1466 2533 V 17 w(V)l(ALID)p 1610 2533 V 16 w(T)l(A)o(G)p 1712 2533 V 17 w(RANGE)75 2589 y FC(en)o(vironmen)o(tal)e(inquiry)h (function,)e(as)g(describ)q(ed)i(in)f(Chapter)f(7.)166 2647 y(The)21 b(con)o(text)f(of)g(the)g(message)g(sen)o(t)g(is)h(sp)q (eci\014ed)i(b)o(y)e(the)f Fv(comm)f FC(argumen)o(t.)35 b(The)20 b(message)75 2704 y(carries)15 b(the)h(con)o(text)e(asso)q (ciated)i(with)f(this)h(comm)o(unicator)f(\(see)g(Chapter)g(5\).)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 25 17 24 bop 75 -100 a Fs(3.3.)34 b(BASIC)16 b(RECEIVE)g(OPERA)l(TION)990 b FC(17)166 45 y(The)14 b(message)f(destination)h(is)h(sp)q(eci\014ed)g (b)o(y)f(the)f Fv(dest)j FC(argumen)o(t)c(as)i(a)f(rank)g(within)i(the) f(pro)q(cess)75 102 y(group)i(asso)q(ciated)h(with)g(that)f(same)h (comm)o(unicator)f(\(remote)g(pro)q(cess)h(group,)f(for)g(in)o(tercomm) o(uni-)75 158 y(cators\).)i(The)d(range)f(of)f(v)m(alid)j(v)m(alues)g (is)e Fo(0,)24 b(...)47 b(,)24 b(n-1)p FC(,)14 b(where)g Fo(n)g FC(is)h(the)f(n)o(um)o(b)q(er)h(of)f(pro)q(cesses)g(in)75 214 y(this)k(group.)25 b(Th)o(us,)17 b(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unications)g(do)f(not)g(use)g(absolute)h(addresses,)f(but)g (only)75 271 y(relativ)o(e)f(ranks)f(within)h(a)f(group.)k(This)d(pro)o (vides)g(imp)q(ortan)o(t)f(mo)q(dularit)o(y)l(.)166 330 y(The)j(message)f(en)o(v)o(elop)q(e)i(w)o(ould)f(normally)g(b)q(e)h (enco)q(ded)g(b)o(y)e(a)g(\014xed-length)j(message)d(header.)75 387 y(Ho)o(w)o(ev)o(er,)c(the)i(actual)f(mec)o(hanism)h(used)g(to)f (asso)q(ciate)g(an)g(en)o(v)o(elop)q(e)i(with)e(a)g(message)g(is)h (implemen-)75 443 y(tation)d(dep)q(enden)o(t:)20 b(some)12 b(of)g(the)g(information)g(\(e.g.,)g(source)g(or)g(destination\))g(ma)o (y)g(b)q(e)h(implicit,)i(and)75 499 y(need)f(not)g(b)q(e)g(explicitly)i (carried)e(b)o(y)g(messages;)f(pro)q(cesses)h(ma)o(y)f(b)q(e)h(iden)o (ti\014ed)i(b)o(y)d(relativ)o(e)h(ranks,)f(or)75 556 y(absolute)j(ids;)f(etc.)75 715 y Ft(3.3)59 b(Basic)19 b(receive)g(op)r(eration)75 822 y FC(The)c(syn)o(tax)g(of)g(the)g (simplest)h(receiv)o(e)g(op)q(eration)g(is)f(giv)o(en)h(b)q(elo)o(w.)75 928 y Fv(MPI)p 160 928 14 2 v 16 w(RECV)g(\(buf,)f(count,)h(datat)o(yp) q(e,)h(source,)e(tag,)g(comm,)e(status\))117 1008 y Fq(OUT)108 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 1089 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1169 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1250 y(IN)155 b Fv(source)468 b Fq(rank)14 b(of)f(source)j(\(in)o (teger\))117 1331 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 1411 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 1492 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\))75 1619 y Fo(int)23 b(MPI)p 245 1619 15 2 v 17 w(Recv\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 1619 V 16 w(Datatype)f(datatype,)g(int)g(source,) 393 1676 y(int)h(tag,)f(MPI)p 683 1676 V 17 w(Comm)g(comm,)g(MPI)p 1034 1676 V 17 w(Status)g(*status\))75 1765 y(MPI)p 150 1765 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS,)g(IERROR\))170 1821 y()g(BUF\(*\))170 1878 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 1878 V 16 w(STATUS)p 1623 1878 V 17 w(SIZE\),)170 1934 y(IERROR)166 2023 y FC(The)15 b(receiv)o(e)h (bu\013er)f(consists)h(of)e(the)h(storage)f(con)o(taining)i Fv(count)g FC(consecutiv)o(e)g(elemen)o(ts)g(of)f(the)75 2080 y(t)o(yp)q(e)21 b(sp)q(eci\014ed)h(b)o(y)f Fv(datat)o(yp)q(e)p FC(,)i(starting)d(at)g(address)g Fv(buf)p FC(.)37 b(The)21 b(length)g(of)f(the)h(receiv)o(ed)g(message)75 2136 y(m)o(ust)15 b(b)q(e)i(less)f(than)g(or)f(equal)i(to)e(the)h(length)g(of)g(the)g (receiv)o(e)g(bu\013er.)22 b(I.e.,)16 b(all)g(incoming)h(data)f(m)o (ust)75 2193 y(\014t,)23 b(without)e(truncation,)i(in)o(to)e(the)h (receiv)o(e)g(bu\013er.)39 b(The)22 b Fv(MPI)p 1262 2193 14 2 v 16 w(PROBE)g FC(function)h(describ)q(ed)g(in)75 2249 y(section)16 b(3.9)e(can)h(b)q(e)h(used)g(to)f(receiv)o(e)h (messages)e(of)h(unkno)o(wn)h(length.)166 2308 y(The)j(selection)h(of)f (a)f(message)h(b)o(y)f(a)h(receiv)o(e)h(op)q(eration)f(is)g(done)g (uniquely)i(according)f(to)e(the)75 2365 y(v)m(alue)i(of)f(the)g (message)g(en)o(v)o(elop)q(e.)32 b(The)19 b(receiv)o(e)h(op)q(eration)g (sp)q(eci\014es)h(an)d Fx(en)o(v)o(elop)q(e)k(pattern)p FC(;)f(a)75 2421 y(message)13 b(can)g(b)q(e)g(receiv)o(ed)h(b)o(y)f (that)g(receiv)o(e)h(op)q(eration)f(only)g(if)h(its)f(en)o(v)o(elop)q (e)h(matc)o(hes)f(that)f(pattern.)75 2478 y(A)19 b(pattern)g(sp)q (eci\014es)i(v)m(alues)g(for)e(the)g Fv(source)p FC(,)i Fv(tag)e FC(and)h Fv(context)g FC(\014elds)h(of)e(the)g(message)g(en)o (v)o(elop)q(e.)75 2534 y(The)d(receiv)o(er)h(ma)o(y)e(sp)q(ecify)j(a)e (wildcard)h Fl(MPI)p 889 2534 13 2 v 14 w(ANY)p 988 2534 V 15 w(SOURCE)d FC(v)m(alue)k(for)d Fv(source)p FC(,)h(and/or)g(a)g (wildcard)75 2591 y Fl(MPI)p 152 2591 V 14 w(ANY)p 251 2591 V 15 w(T)m(A)o(G)c FC(v)m(alue)j(for)d Fv(tag)p FC(,)i(indicating)h(that)d(an)o(y)h(source)h(and/or)e(tag)h(are)g (acceptable.)20 b(It)13 b(cannot)75 2647 y(sp)q(ecify)h(a)f(wildcard)h (v)m(alue)g(for)f Fv(context)p FC(.)20 b(Th)o(us,)13 b(a)g(message)f(can)i(b)q(e)f(receiv)o(ed)h(b)o(y)f(a)g(receiv)o(e)h (op)q(eration)75 2704 y(only)23 b(if)g(it)g(is)g(addressed)h(to)e(the)g (receiving)j(pro)q(cess,)f(has)f(a)f(matc)o(hing)h(con)o(text,)g(has)g (matc)o(hing)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 18 26 18 25 bop 75 -100 a FC(18)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FC(source)e(unless)g (source=)p Fl(MPI)p 581 45 13 2 v 15 w(ANY)p 681 45 V 15 w(SOURCE)d FC(in)k(the)e(pattern,)g(and)h(has)g(a)f(matc)o(hing)g (tag)g(unless)i(tag=)75 102 y Fl(MPI)p 152 102 V 14 w(ANY)p 251 102 V 15 w(T)m(A)o(G)f FC(in)i(the)f(pattern.)166 159 y(The)d(message)g(tag)f(is)i(sp)q(eci\014ed)h(b)o(y)e(the)g Fv(tag)h FC(argumen)o(t)e(of)h(the)g(receiv)o(e)h(op)q(eration.)19 b(The)12 b(message)75 215 y Fv(context)20 b FC(is)f(the)f(con)o(text)g (attac)o(hed)g(with)h(the)f(comm)o(unicator)g(sp)q(eci\014ed)j(b)o(y)d (the)h(argumen)o(t)e Fv(comm)p FC(.)75 272 y(The)g(message)g(source,)h (if)f(di\013eren)o(t)h(from)32 b Fl(MPI)p 929 272 V 14 w(ANY)p 1028 272 V 15 w(SOURCE)p FC(,)15 b(is)j(sp)q(eci\014ed)h(as)e (a)g(rank)f(within)j(the)75 328 y(pro)q(cess)g(group)f(asso)q(ciated)h (with)g(that)e(same)h(comm)o(unicator)h(\(remote)e(pro)q(cess)i(group,) g(for)f(in)o(ter-)75 385 y(comm)o(unicators\).)25 b(Th)o(us,)18 b(the)f(range)g(of)g(v)m(alid)i(v)m(alues)f(for)f(the)g Fv(source)h FC(argumen)o(t)f(is)h FA(f)23 b Fo(0,)h(...)47 b(,)75 441 y(n-1)23 b FA(g[f)h Fo(MPI)p 345 441 15 2 v 17 w(ANY)p 434 441 V 17 w(SOURCE)f FA(g)p FC(,)14 b(where)i Fo(n)f FC(is)g(the)h(n)o(um)o(b)q(er)f(of)g(pro)q(cesses)h(in)g(this)f (group.)166 498 y(Note)j(the)i(asymmetry)d(b)q(et)o(w)o(een)j(send)f (and)g(receiv)o(e)h(op)q(erations:)28 b(A)19 b(receiv)o(e)h(op)q (eration)f(ma)o(y)75 555 y(accept)13 b(messages)f(from)g(an)h (arbitrary)f(sender;)i(on)f(the)g(other)f(hand,)i(a)e(send)h(op)q (eration)g(m)o(ust)g(sp)q(ecify)75 611 y(a)i(unique)j(receiv)o(er.)k (This)16 b(matc)o(hes)f(a)h(\\push")f(comm)o(unication)i(mec)o(hanism,) f(where)g(data)f(transfer)75 668 y(is)i(e\013ected)h(b)o(y)f(the)g (sender)g(\(rather)f(than)h(a)g(\\pull")h(mec)o(hanism,)g(where)f(data) f(transfer)g(is)i(e\013ected)75 724 y(b)o(y)d(the)g(receiv)o(er\).)166 781 y(Source)h(=)f(destination)h(is)g(allo)o(w)o(ed:)k(a)15 b(pro)q(cess)g(can)h(send)g(a)e(message)h(to)g(itself.)75 908 y Fp(3.3.1)49 b(Return)16 b(status)75 995 y FC(The)e(source)g(or)f (the)h(tag)f(of)g(a)h(receiv)o(ed)h(message)e(ma)o(y)g(not)h(b)q(e)g (kno)o(wn)g(if)g(wildcard)h(v)m(alues)g(w)o(ere)e(used)75 1051 y(in)h(the)f(receiv)o(e)h(op)q(eration.)19 b(Also,)13 b(the)g(actual)g(length)h(of)f(the)g(message)f(receiv)o(ed)i(ma)o(y)e (not)h(b)q(e)h(kno)o(wn.)75 1108 y(Th)o(us,)j(this)h(information)f (needs)h(to)e(b)q(e)i(returned)g(b)o(y)f(the)g(receiv)o(e)h(op)q (eration.)26 b(The)17 b(information)g(is)75 1164 y(returned)12 b(b)o(y)g(the)f Fv(status)j FC(argumen)o(t)d(of)g(the)h Fv(MPI)p 923 1164 14 2 v 16 w(RECV)g FC(function.)20 b(This)12 b(is)g(an)g(argumen)o(t)f(of)g(a)g(sp)q(ecial)75 1221 y(MPI-de\014ned)19 b(t)o(yp)q(e.)27 b(Status)17 b(v)m(ariables)i(need)f(to)f(b)q(e)h(explicitly)i(allo)q(cated)f(b)o(y) e(the)h(user)g({)f(they)g(are)75 1277 y(not)e(system)g(ob)s(jects.)166 1334 y(The)f(status)f(v)m(ariable)i(can)f(b)q(e)g(\\deco)q(ded")h(to)e (retriev)o(e)h(the)g Fv(count)p FC(,)g Fv(source)h FC(and)f Fv(tag)f FC(\014elds,)i(using)75 1391 y(the)g(query)g(functions)h (listed)g(b)q(elo)o(w.)k(The)15 b(information)g(returned)h(b)o(y)f (these)g(query)g(functions)g(is)h(the)75 1447 y(information)h(last)g (stored)g(in)h(the)f Fv(status)i FC(v)m(ariable)f(b)o(y)f(a)g(receiv)o (e)h(function.)26 b(It)17 b(is)h(erroneous)e(to)h(call)75 1504 y(these)e(query)h(functions)g(if)f(the)h Fv(status)h FC(v)m(ariable)f(w)o(as)f(nev)o(er)g(set)g(b)o(y)g(a)g(receiv)o(e.)75 1608 y Fv(MPI)p 160 1608 V 16 w(GET)p 264 1608 V 17 w(SOURCE\(status,)j (source\))117 1686 y Fq(IN)155 b Fv(status)476 b Fq(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 1763 y(OUT)108 b Fv(source)468 b Fq(source)15 b(rank)f(\(in)o(teger\)) 75 1888 y Fo(int)23 b(MPI)p 245 1888 15 2 v 17 w(Get)p 334 1888 V 17 w(source\(MPI)p 591 1888 V 16 w(Status)g(status,)g(int)g (*source\))75 1975 y(MPI)p 150 1975 V 17 w(GET)p 239 1975 V 17 w(SOURCE\(STATUS,)e(SOURCE,)i(IERROR\))170 2032 y(INTEGER)g(STATUS\(MPI)p 604 2032 V 16 w(STATUS)p 764 2032 V 17 w(SIZE\),)g(SOURCE,)f(IERROR)166 2119 y FC(Returns)13 b(the)g(rank)g(of)f(the)h(message)f(source)h(in)h(the)f (group)f(asso)q(ciated)h(with)g(the)g(comm)o(unicator)75 2176 y(that)h(w)o(as)h(used)h(to)e(receiv)o(e)i(\(remote)f(group,)f (for)h(in)o(tercomm)o(unicators\).)75 2280 y Fv(MPI)p 160 2280 14 2 v 16 w(GET)p 264 2280 V 17 w(T)l(A)o(G\(status,)i(tag\)) 117 2358 y Fq(IN)155 b Fv(status)476 b Fq(return)15 b(status)g(of)e (receiv)o(e)i(op)q(eration)f(\(Status\))117 2435 y(OUT)108 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))75 2560 y Fo(int)23 b(MPI)p 245 2560 15 2 v 17 w(Get)p 334 2560 V 17 w(tag\(MPI)p 519 2560 V 16 w(Status)g(status,)g(int)h(*tag\)) 75 2647 y(MPI)p 150 2647 V 17 w(GET)p 239 2647 V 17 w(TAG\(STATUS,)e (TAG,)h(IERROR\))170 2704 y(INTEGER)g(STATUS\(MPI)p 604 2704 V 16 w(STATUS)p 764 2704 V 17 w(SIZE\),)g(TAG,)g(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 19 27 19 26 bop 75 -100 a Fs(3.4.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b FC(19)166 45 y(Returns)16 b(the)f(tag)f(of)h (receiv)o(ed)h(message.)75 150 y Fv(MPI)p 160 150 14 2 v 16 w(GET)p 264 150 V 17 w(COUNT\(status,)h(datat)o(yp)q(e,)f (count\))117 229 y Fq(IN)155 b Fv(status)476 b Fq(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\))117 307 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)e(\(handle\))117 385 y(OUT)108 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(elemen)o(ts)f(\(in)o(teger\))75 511 y Fo(int)23 b(MPI)p 245 511 15 2 v 17 w(Get)p 334 511 V 17 w(count\(MPI)p 567 511 V 16 w(Status)g(status,)g(MPI)p 1013 511 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 598 y(MPI)p 150 598 V 17 w(GET)p 239 598 V 17 w(COUNT\(STATUS,)e (DATATYPE,)g(COUNT,)h(IERROR\))170 655 y(INTEGER)g(STATUS\(MPI)p 604 655 V 16 w(STATUS)p 764 655 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 743 y FC(Returns)15 b(the)f(n)o(um)o(b)q(er)h(of)f(elemen)o (ts)h(receiv)o(ed.)21 b(\(Here,)14 b(again,)h(w)o(e)f(coun)o(t)g Fr(elements)p FC(,)f(not)h Fr(bytes)p FC(.\))75 799 y(The)19 b Fv(datat)o(yp)q(e)h FC(argumen)o(t)e(should)i(matc)o(h)e(the)g (argumen)o(t)g(pro)o(vided)i(b)o(y)e(the)h(receiv)o(e)g(call)h(that)e (set)75 856 y(the)d Fv(status)i FC(v)m(ariable.)166 914 y(Note)e(that)g(it)h(is)g(not)f(mandatory)g(to)g(query)g(the)h(return)g (status)e(after)h(a)g(receiv)o(e.)22 b(The)16 b(receiv)o(er)75 970 y(will)h(use)e(these)h(calls,)g(or)e(some)h(of)g(them,)g(only)h (when)f(the)h(information)f(they)g(return)g(is)h(needed.)166 1104 y Fn(Implemen)o(tati)o(on)c(note:)166 1162 y Fq(One)j(exp)q(ects)g (deco)q(de)h(functions)e(to)f(b)q(e)i(in-lined)e(in)g(man)o(y)f (implemen)o(tations.)166 1302 y FC(All)19 b(send)g(and)g(receiv)o(e)g (op)q(erations)f(use)h(the)f Fv(buf)p FC(,)h Fv(count)p FC(,)h Fv(datat)o(yp)q(e)p FC(,)g Fv(source)p FC(,)f Fv(dest)p FC(,)h Fv(tag)p FC(,)f Fv(comm)75 1359 y FC(and)e Fv(status)i FC(argumen)o(ts)d(in)i(the)f(same)g(w)o(a)o(y)f(as)g(the)h (basic)h Fv(MPI)p 1201 1359 14 2 v 16 w(SEND)f FC(and)g Fv(MPI)p 1523 1359 V 16 w(RECV)h FC(op)q(erations)75 1415 y(describ)q(ed)f(in)f(this)g(section.)75 1567 y Ft(3.4)59 b(Semantics)18 b(of)i(p)r(oint-to-p)r(oint)e(communication)75 1671 y FC(A)g(v)m(alid)h(MPI)e(implemen)o(tation)i(guaran)o(tees)e (certain)h(general)g(prop)q(erties)g(of)f(p)q(oin)o(t-to-p)q(oin)o(t)h (com-)75 1727 y(m)o(unication,)e(whic)o(h)g(are)f(describ)q(ed)i(in)f (this)f(section.)75 1855 y Fv(Order)46 b FC(Messages)14 b(are)h Fr(non-overtaking)p FC(,)g(within)h(eac)o(h)f(con)o(text:)20 b(if)c(t)o(w)o(o)d(messages)i(are)g(successiv)o(ely)75 1912 y(sen)o(t)10 b(from)g(the)g(same)g(source,)h(to)e(the)i(same)f (destination,)h(with)g(the)f(same)g(con)o(text,)h(then)f(the)g (messages)75 1968 y(are)17 b(receiv)o(ed)i(in)f(the)f(order)g(they)h(w) o(ere)f(sen)o(t.)26 b(I.e.,)17 b(if)h(the)f(receiv)o(er)i(p)q(osts)e(t) o(w)o(o)f(successiv)o(e)i(receiv)o(es)75 2025 y(that)g(b)q(oth)g(matc)o (h)g(either)i(message,)e(then)h(the)f(\014rst)h(receiv)o(e)g(will)h (receiv)o(e)g(the)e(\014rst)g(message,)h(and)75 2081 y(the)e(second)h(receiv)o(e)h(will)g(receiv)o(e)f(the)f(second)h (message.)26 b(This)18 b(requiremen)o(t)g(facilitates)g(matc)o(hing)75 2138 y(of)e(sends)h(to)f(receiv)o(es.)25 b(It)16 b(guaran)o(tees)g (that)g(message-passing)g(co)q(de)h(is)g(deterministic,)i(if)e(pro)q (cesses)75 2194 y(are)e(single-threaded)i(and)e(wildcard)i Fl(MPI)p 817 2194 13 2 v 14 w(ANY)p 916 2194 V 15 w(SOURCE)c FC(is)j(not)f(used)h(in)g(receiv)o(es.)166 2252 y(If)h(a)g(pro)q(cess)h (has)f(a)g(single)h(thread)f(of)g(execution,)i(then)e(an)o(y)g(t)o(w)o (o)f(comm)o(unications)i(executed)75 2308 y(b)o(y)e(this)h(pro)q(cess)g (are)f(ordered.)23 b(On)17 b(the)f(other)g(hand,)h(if)g(the)f(pro)q (cess)h(is)g(m)o(ulti-threaded,)g(then)g(the)75 2365 y(seman)o(tics)e(of)f(thread)h(execution)h(ma)o(y)e(not)g(de\014ne)i(a) e(relativ)o(e)i(order)e(b)q(et)o(w)o(een)h(t)o(w)o(o)e(send)j(op)q (erations)75 2421 y(executed)g(b)o(y)f(t)o(w)o(o)f(distinct)j(threads:) i(the)d(op)q(erations)f(are)g(logically)i(concurren)o(t,)e(ev)o(en)h (though)f(one)75 2478 y(ph)o(ysically)22 b(precedes)e(the)g(other.)33 b(In)20 b(suc)o(h)g(a)f(case,)i(the)e(t)o(w)o(o)g(messages)g(sen)o(t)g (can)h(b)q(e)g(receiv)o(ed)h(in)75 2534 y(an)o(y)f(order.)37 b(Similarly)l(,)24 b(if)d(t)o(w)o(o)e(receiv)o(e)j(op)q(erations)f (that)f(are)g(logically)j(concurren)o(t)e(receiv)o(e)g(t)o(w)o(o)75 2591 y(successiv)o(ely)i(sen)o(t)d(messages,)h(then)g(the)g(t)o(w)o(o)f (messages)g(can)h(matc)o(h)f(the)h(t)o(w)o(o)e(receiv)o(es)j(in)f (either)75 2647 y(order.)31 b(Note)19 b(that)f(the)h(receiv)o(er)h(is)g (not)e(required)i(to)f(receiv)o(e)h(messages)e(in)i(the)f(order)g(they) g(w)o(ere)75 2704 y(sen)o(t,)c(if)g(they)h(ha)o(v)o(e)e(distinct)j(en)o (v)o(elop)q(es.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 28 20 27 bop 75 -100 a FC(20)613 b Fs(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fv(Progress)45 b FC(If)13 b(a)g(pair)g(of)f(matc)o(hing)h(send)h(and)f(receiv)o(es)g (ha)o(v)o(e)g(b)q(een)h(initiated)g(on)f(t)o(w)o(o)e(pro)q(cesses,)j (then)75 102 y(at)k(least)h(one)h(of)e(these)h(t)o(w)o(o)f(op)q (erations)h(will)i(complete,)f(indep)q(enden)o(tly)i(of)c(other)h (actions)g(in)h(the)75 158 y(system:)e(the)13 b(send)h(op)q(eration)f (will)h(complete,)g(unless)g(the)f(receiv)o(e)h(is)f(satis\014ed)h(b)o (y)e(another)h(message,)75 214 y(and)h(completes;)g(the)g(receiv)o(e)h (op)q(eration)f(will)h(complete,)g(unless)g(the)f(message)f(sen)o(t)g (is)i(consumed)f(b)o(y)75 271 y(another)h(matc)o(hing)g(receiv)o(e)h (that)f(w)o(as)f(p)q(osted)h(at)g(the)g(same)g(destination)h(pro)q (cess.)75 391 y Fv(F)o(airness)46 b FC(MPI)15 b(mak)o(es)g(no)g(guaran) o(tee)g(of)g Fr(fairness)f FC(in)i(the)g(handling)h(of)e(comm)o (unication.)21 b(Supp)q(ose)75 447 y(that)c(a)f(send)i(w)o(as)f(p)q (osted.)26 b(Then)18 b(it)f(is)h(p)q(ossible)h(that)e(the)g (destination)h(pro)q(cess)g(rep)q(eatedly)g(p)q(osts)75 503 y(a)d(receiv)o(e)h(that)f(matc)o(hes)f(this)i(send,)g(y)o(et)e(the) i(message)f(is)g(nev)o(er)h(receiv)o(ed,)g(b)q(ecause)g(it)g(is)f(eac)o (h)h(time)75 560 y(o)o(v)o(ertak)o(en)e(b)o(y)i(another)f(message,)g (sen)o(t)h(from)f(another)g(source.)21 b(Similarly)l(,)d(supp)q(ose)f (that)e(a)g(receiv)o(e)75 616 y(w)o(as)f(p)q(osted)i(b)o(y)f(a)g(m)o (ulti-threaded)h(pro)q(cess.)k(Then)c(it)f(is)h(p)q(ossible)h(that)d (messages)h(that)f(matc)o(h)h(this)75 673 y(receiv)o(e)20 b(are)g(rep)q(eatedly)g(receiv)o(ed,)i(y)o(et)d(the)g(receiv)o(e)i(is)f (nev)o(er)f(satis\014ed,)i(b)q(ecause)f(it)g(is)g(o)o(v)o(ertak)o(en)75 729 y(b)o(y)15 b(other)h(receiv)o(es)g(p)q(osted)g(at)e(this)i(no)q(de) h(\(b)o(y)e(other)g(executing)h(threads\).)21 b(It)15 b(is)h(the)g(programmer's)75 786 y(resp)q(onsibilit)o(y)i(to)c(prev)o (en)o(t)h(starv)m(ation)g(in)h(suc)o(h)g(situations.)75 905 y Fv(Resource)e(limitations)45 b FC(The)13 b(curren)o(t)f(practice) h(for)f(man)o(y)g(commercial)i(message-passing)e(libraries)i(is)75 962 y(that)f(\(short\))f(messages)h(are)g(bu\013ered)g(b)o(y)h(the)f (system,)g(th)o(us)g(allo)o(wing)h(blo)q(c)o(king)h(send)f(op)q (erations)g(to)75 1018 y(complete)d(ahead)f(of)g(the)h(matc)o(hing)f (receiv)o(es.)19 b(It)11 b(is)f(exp)q(ected)i(that)e(man)o(y)f(MPI)i (implemen)o(tations)g(will)75 1075 y(follo)o(w)j(this)g(practice,)g (and)f(pro)o(vide)h(the)g(same)f(lev)o(el)i(of)e(bu\013ering)h(that)f (is)h(a)o(v)m(ailable)h(on)f(the)f(libraries)75 1131 y(they)21 b(replace.)37 b(Ho)o(w)o(ev)o(er,)20 b(message)g(bu\013ering) h(is)h(not)e(a)g(univ)o(ersal)i(practice.)36 b(Ev)o(en)21 b(on)f(systems)75 1188 y(where)e(bu\013ering)g(o)q(ccur,)g(the)f(amoun) o(t)g(of)g(bu\013er)h(space)f(a)o(v)m(ailable)j(and)d(the)h(w)o(a)o(y)e (it)i(is)g(allo)q(cated)g(is)75 1244 y(b)q(ound)e(to)f(b)q(e)h (implemen)o(tation)g(dep)q(enden)o(t.)166 1300 y(Therefore,)d(message)h (bu\013ering)g(is)g(not)f(mandated)h(b)o(y)f(MPI.)h(A)f(v)m(alid)j(MPI) d(implemen)o(tation)i(of)75 1357 y Fv(MPI)p 160 1357 14 2 v 16 w(SEND)j FC(is)g(to)f(blo)q(c)o(k)h(the)f(sender)i(un)o(til)f (a)f(matc)o(hing)h(receiv)o(e)g(has)g(b)q(een)g(initiated.)29 b(In)18 b(general,)75 1413 y(the)d(programmer)e(can)i(mak)o(e)g(no)f (assumptions)h(on)g(the)g(a)o(v)m(ailabilit)o(y)i(of)d(bu\013er)h (space,)f(and)h(ho)o(w)g(this)75 1470 y(space)g(is)g(allo)q(cated.)21 b(Th)o(us,)14 b(p)q(ortable)h(\(safe\))f(MPI)g(co)q(de)i(should)f(w)o (ork)f(under)h(the)g(assumption)g(that)75 1526 y(an)g(arbitrary)g (subset)h(of)f(the)h(send)g(op)q(erations)f(are)h(going)f(to)g(return)g (b)q(efore)h(a)f(matc)o(hing)h(receiv)o(e)g(is)75 1583 y(p)q(osted,)f(and)g(the)h(rest)f(will)h(blo)q(c)o(k)g(un)o(til)h(a)e (matc)o(hing)g(receiv)o(e)h(is)g(p)q(osted.)166 1639 y(An)g(MPI)g(implemen)o(tation)h(will)g(pro)o(vide)f(information)g(and) g(con)o(trol)f(on)h(the)g(amoun)o(t)f(of)g(a)o(v)m(ail-)75 1696 y(able)i(bu\013er)g(space)g(and)g(on)f(the)h(bu\013ering)g(p)q (olicy)h(via)f(the)g(en)o(vironmen)o(tal)g(enquiries)i(describ)q(ed)f (in)75 1752 y(Chapter)d(7.)166 1809 y(Examples)h(\(in)o(v)o(olving)g(t) o(w)o(o)e(pro)q(cesses)h(with)h(ranks)f(0)f(and)i(1\))166 1865 y(The)f(follo)o(wing)h(program)e(is)i(safe,)f(and)g(should)h(alw)o (a)o(ys)f(succeed.)75 2013 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,) h(ierr\))75 2069 y(IF)h(\(rank.EQ.0\))123 2126 y(THEN)170 2182 y(CALL)g(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(ierr\))170 2239 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(1,)h(tag,)f(comm,)g(status,)g(ierr\))123 2295 y(ELSE)95 b(!)23 b(rank.EQ.1)170 2352 y(CALL)h(MPI_RECV\(recvbuf,) d(count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))170 2408 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))75 2464 y(END)g(IF)166 2556 y FC(The)15 b(follo)o(wing)h(program)e(is)i(erroneous,)f(and)g(should)h(alw)o(a)o (ys)f(fail.)75 2704 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 29 21 28 bop 75 -100 a Fs(3.5.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) 1105 b FC(21)75 45 y Fo(IF)24 b(\(rank.EQ.0\))123 102 y(THEN)170 158 y(CALL)g(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h (tag,)f(comm,)g(status,)g(ierr\))170 214 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))123 271 y(ELSE)95 b(!)23 b(rank.EQ.1)170 327 y(CALL)h(MPI_RECV\(recvbuf,)d (count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))170 384 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))75 440 y(END)g(IF)166 538 y FC(The)15 b(receiv)o(e)g(op)q(eration)g(of)f(the)h(\014rst)f(pro)q(cess)h(m)o (ust)f(complete)i(b)q(efore)f(its)f(send,)h(and)g(can)g(com-)75 594 y(plete)j(only)g(if)g(the)f(matc)o(hing)g(send)h(of)f(the)g(second) h(pro)q(cessor)f(is)h(executed;)g(the)g(receiv)o(e)g(op)q(eration)75 651 y(of)c(the)h(second)g(pro)q(cess)g(m)o(ust)f(complete)h(b)q(efore)g (its)f(send)i(and)e(can)h(complete)g(only)g(if)g(the)g(matc)o(hing)75 707 y(send)h(of)f(the)g(\014rst)g(pro)q(cess)g(is)h(executed.)21 b(This)15 b(program)f(will)j(deadlo)q(c)o(k.)166 765 y(The)10 b(follo)o(wing)h(program)f(is)g(unsafe,)h(and)g(ma)o(y)e (succeed)j(or)e(fail,)h(dep)q(ending)i(on)d(implemen)o(tation.)75 933 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 990 y(IF)h(\(rank.EQ.0\))123 1046 y(THEN)170 1103 y(CALL)g (MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\)) 170 1159 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))123 1216 y(ELSE)95 b(!)23 b(rank.EQ.1)170 1272 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 1329 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))75 1385 y(END)g(IF)166 1553 y FC(The)11 b(message)g(sen)o(t)g(b)o(y)g(eac) o(h)g(pro)q(cess)g(has)g(to)f(b)q(e)i(copied)g(out)f(b)q(efore)g(the)g (send)h(op)q(eration)f(returns)75 1609 y(and)k(the)g(receiv)o(e)h(op)q (eration)f(starts.)k(F)l(or)14 b(the)h(program)f(to)g(complete,)h(it)h (is)f(necessary)g(that)g(at)f(least)75 1665 y(one)i(of)g(the)h(t)o(w)o (o)d(messages)i(sen)o(t)g(is)h(bu\013ered.)23 b(Th)o(us,)16 b(this)h(program)e(can)i(succeed)g(only)g(if)g(the)f(MPI)75 1722 y(implemen)o(tation)e(bu\013ers)f(messages,)g(and)g(the)g(comm)o (unication)h(system)f(has)g(su\016cien)o(t)h(bu\013er)f(space)75 1778 y(to)i(bu\013er)g Fv(count)h FC(w)o(ords)f(of)f(data.)75 1927 y Ft(3.5)59 b(Data)19 b(T)-5 b(yp)r(e)19 b(Matching)75 2031 y FC(One)d(can)f(think)h(of)f(message)g(transmission)g(as)g (consisting)h(of)f(three)g(phases:)131 2128 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i(send)f(bu\013er)h(and)f(a)g(message)g (is)g(assem)o(bled)131 2226 y(2.)22 b(A)15 b(message)g(is)g (transferred)g(from)f(sender)i(to)f(receiv)o(er)131 2324 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h(incoming)i(message)e (and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o(e)h(bu\013er)166 2421 y(T)o(yp)q(e)k(matc)o(hing)f(has)h(to)e(b)q(e)j(observ)o(ed)e(at)g (eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f (eac)o(h)75 2478 y(v)m(ariable)i(in)f(the)g(sender)g(bu\013er)g(has)g (to)f(matc)o(h)g(the)g(t)o(yp)q(e)h(sp)q(eci\014ed)i(for)d(that)g(en)o (try)g(b)o(y)h(the)f(send)75 2534 y(op)q(eration;)d(the)f(t)o(yp)q(e)h (sp)q(eci\014ed)i(b)o(y)e(the)f(send)i(op)q(eration)e(has)h(to)f(matc)o (h)g(the)h(t)o(yp)q(e)f(sp)q(eci\014ed)j(b)o(y)e(the)75 2591 y(receiv)o(e)g(op)q(eration;)e(and)h(the)g(t)o(yp)q(e)f(of)h(eac)o (h)f(v)m(ariable)i(in)g(the)e(receiv)o(e)i(bu\013er)f(has)f(to)g(matc)o (h)g(the)h(t)o(yp)q(e)75 2647 y(sp)q(eci\014ed)f(for)e(that)g(en)o(try) g(b)o(y)g(the)g(receiv)o(e)h(op)q(eration.)20 b(A)12 b(program)f(that)h(fails)h(to)e(observ)o(e)h(these)h(three)75 2704 y(rules)j(is)g(erroneous.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 30 22 29 bop 75 -100 a FC(22)613 b Fs(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(T)l(o)f (de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l(,)j(w)o(e)d (need)i(to)e(deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 102 y(t)o(yp)q(es)e(of)g(the)h(host)f(language)g(with)h(t)o(yp) q(es)g(sp)q(eci\014ed)h(in)f(comm)o(unication)h(op)q(erations;)e(and)h (matc)o(hing)75 158 y(of)h(t)o(yp)q(es)g(at)g(sender)g(and)h(receiv)o (er.)166 214 y(A)h(t)o(yp)q(e)f(sp)q(eci\014ed)j(for)d(an)h(en)o(try)f (b)o(y)h(a)f(send)h(op)q(eration)g(matc)o(hes)f(the)h(t)o(yp)q(e)g(sp)q (eci\014ed)i(for)d(that)75 271 y(en)o(try)11 b(b)o(y)h(a)f(receiv)o(e)h (op)q(eration)g(if)g(b)q(oth)f(op)q(erations)h(use)g(iden)o(tical)h (names:)36 b Fv(MPI)p 1500 271 14 2 v 16 w(INTEGER)12 b FC(matc)o(hes)75 327 y Fv(MPI)p 160 327 V 16 w(INTEGER)p FC(,)j Fv(MPI)p 475 327 V 16 w(REAL)h FC(matc)o(hes)30 b Fv(MPI)p 891 327 V 15 w(REAL)p FC(,)16 b(and)f(so)g(on.)166 384 y(The)k(t)o(yp)q(e)g(of)f(a)g(v)m(ariable)i(in)g(a)e(host)h (program)e(matc)o(hes)h(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)i(in)e(the)g (comm)o(u-)75 440 y(nication)g(op)q(eration)g(if)f(the)h(datat)o(yp)q (e)f(name)g(used)h(b)o(y)f(that)f(op)q(eration)i(corresp)q(onds)f(to)g (the)g(basic)75 497 y(t)o(yp)q(e)i(of)g(the)g(host)g(program)f(v)m (ariable:)31 b(an)20 b(en)o(try)f(with)i(t)o(yp)q(e)f(name)40 b Fv(MPI)p 1449 497 V 16 w(INTEGER)21 b FC(matc)o(hes)e(a)75 553 y(F)l(ortran)c(v)m(ariable)j(of)f(t)o(yp)q(e)f Fo(INTEGER)p FC(,)g(an)g(en)o(try)g(with)h(t)o(yp)q(e)g(name)g Fv(MPI)p 1368 553 V 16 w(REAL)g FC(matc)o(hes)f(a)g(F)l(ortran)75 610 y(v)m(ariable)j(of)e(t)o(yp)q(e)g Fo(REAL)p FC(,)f(and)i(so)f(on.) 26 b(There)18 b(is)f(one)h(exception)g(to)f(this)h(last)f(rule:)25 b(An)18 b(en)o(try)f(with)75 666 y(t)o(yp)q(e)h(name)37 b Fv(MPI)p 408 666 V 15 w(BYTE)19 b FC(can)f(b)q(e)h(used)g(to)e(matc)o (h)h(an)o(y)g(b)o(yte)g(of)g(storage)e(\(on)i(a)g(b)o(yte-addressable) 75 723 y(mac)o(hine\),)d(irresp)q(ectiv)o(e)i(of)d(the)h(datat)o(yp)q (e)g(of)f(the)h(v)m(ariable)i(that)d(con)o(tains)h(this)g(b)o(yte.)20 b(The)15 b(v)m(alue)h(of)75 779 y(the)f(message)g(en)o(try)g(will)i(b)q (e)f(the)f(binary)h(v)m(alue)g(of)f(the)g(corresp)q(onding)h(b)o(yte)f (in)h(memory)l(.)166 835 y(W)l(e)f(th)o(us)g(ha)o(v)o(e)g(t)o(w)o(o)f (cases:)143 925 y FA(\017)23 b FC(Comm)o(unication)11 b(of)f(t)o(yp)q(ed)i(v)m(alues)g(\(e.g.,)e(with)h(datat)o(yp)q(e)g (di\013eren)o(t)g(from)21 b Fv(MPI)p 1586 925 V 16 w(BYTE)p FC(\),)11 b(where)189 981 y(the)k(datat)o(yp)q(es)f(of)g(the)h(corresp) q(onding)h(en)o(tries)f(in)h(the)f(sender)g(program,)e(in)j(the)f(send) g(call,)h(in)189 1037 y(the)f(receiv)o(e)h(call)g(and)g(in)g(the)f (receiv)o(er)h(program)e(should)i(all)g(matc)o(h.)143 1129 y FA(\017)23 b FC(Comm)o(unication)16 b(of)g(un)o(t)o(yp)q(ed)h(v) m(alues)g(\(e.g.,)e(of)g(datat)o(yp)q(e)h Fv(MPI)p 1330 1129 V 16 w(BYTE)p FC(\),)g(where)g(b)q(oth)g(sender)189 1186 y(and)g(receiv)o(er)h(use)f(the)g(datat)o(yp)q(e)g Fv(MPI)p 881 1186 V 15 w(BYTE)p FC(.)h(In)f(this)h(case,)f(there)g(are) g(no)f(requiremen)o(ts)i(on)189 1242 y(the)d(t)o(yp)q(es)h(of)f(the)g (corresp)q(onding)h(en)o(tries)g(in)h(the)e(sender)h(and)g(the)f (receiv)o(er)h(programs,)e(nor)h(is)189 1299 y(it)h(required)h(that)f (they)g(b)q(e)h(the)f(same.)166 1388 y(The)g(follo)o(wing)h(examples)g (illustrate)h(these)e(t)o(w)o(o)f(cases.)166 1444 y(First)h(program:)75 1533 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1590 y(IF\(rank.EQ.0\))123 1646 y(THEN)170 1703 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 1759 y(ELSE)170 1815 y(CALL)h(MPI_RECV\(a\(1\),)e(15,)h(MPI_REAL,)g(0,) g(tag,)h(comm,)f(status,)g(ierr\))75 1872 y(END)g(IF)166 1961 y FC(This)c(co)q(de)f(is)h(correct)e(if)i(b)q(oth)f(sender)h(and)f (receiv)o(er)h(programs)d(ha)o(v)o(e)i(allo)q(cated)h(consecutiv)o(e)75 2017 y(storage)14 b(for)h(ten)g(real)g(n)o(um)o(b)q(ers,)g(starting)g (from)f Fo(a\(1\))p FC(.)166 2074 y(Second)i(program:)75 2163 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2219 y(IF\(rank.EQ.0\))123 2276 y(THEN)170 2332 y(CALL)h (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))123 2389 y(ELSE)170 2445 y(CALL)h(MPI_RECV\(a\(1\),)e(40,)h(MPI_BYTE,)g(0,) g(tag,)h(comm,)f(status,)g(ierr\))75 2502 y(END)g(IF)166 2591 y FC(This)d(co)q(de)g(is)f(erroneous,)h(since)g(sender)g(and)g (receiv)o(er)g(do)f(not)f(pro)o(vide)i(matc)o(hing)f(datat)o(yp)q(e)75 2647 y(argumen)o(ts.)166 2704 y(Third)d(program:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 31 23 30 bop 75 -100 a Fs(3.6.)34 b(D)o(A)l(T)l(A)15 b(CONVERSION)1197 b FC(23)75 45 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 102 y(IF\(rank.EQ.0\))123 158 y(THEN)170 214 y(CALL)h (MPI_SEND\(a\(1\),)e(40,)h(MPI_BYTE,)g(1,)g(tag,)h(comm,)f(ierr\))123 271 y(ELSE)170 327 y(CALL)h(MPI_RECV\(a\(1\),)e(60,)h(MPI_BYTE,)g(0,)g (tag,)h(comm,)f(status,)g(ierr\))75 384 y(END)g(IF)166 478 y FC(This)16 b(co)q(de)f(is)h(correct,)f(irresp)q(ectiv)o(e)h(of)f (the)g(t)o(yp)q(e)h(of)e Fo(a\(1\))h FC(and)g(follo)o(wing)h(v)m (ariables)h(in)f(store.)166 535 y(Some)e(implemen)o(tations)g(ma)o(y)f (disallo)o(w)i(the)f(transfer)e(of)i(F)l(ortran)e(c)o(haracter)h (strings)h(using)g(the)75 592 y(MPI)h(t)o(yp)q(e)g Fv(MPI)p 366 592 14 2 v 16 w(BYTE)p FC(.)75 736 y Ft(3.6)59 b(Data)19 b(conversion)75 838 y FC(One)d(of)f(the)g(goals)g(of)g(MPI)g(is)h(to)e (supp)q(ort)i(parallel)g(computations)f(across)g(heterogeneous)g(en)o (viron-)75 894 y(men)o(ts.)20 b(Comm)o(unication)15 b(in)h(a)f (heterogeneous)g(en)o(vironmen)o(t)h(ma)o(y)e(require)i(data)f(con)o(v) o(ersions.)166 951 y(W)l(e)g(use)h(the)f(follo)o(wing)h(terminology:)75 1046 y Fx(t)o(yp)q(e)h(con)o(v)o(ersion)22 b FC(c)o(hanges)16 b(the)h(datat)o(yp)q(e)f(of)g(a)g(v)m(alue,)h(e.g.)23 b(b)o(y)17 b(rounding)g(a)f Fv(REAL)h FC(to)f(an)g Fv(INTE-)189 1102 y(GER)p FC(.)75 1197 y Fx(represen)o(tation)h(con)o(v)o(ersion)22 b FC(c)o(hanges)16 b(the)h(binary)g(represen)o(tation)f(of)g(a)g(v)m (alue,)h(e.g.)23 b(from)16 b(Hex)189 1253 y(\015oating)f(p)q(oin)o(t)g (to)g(IEEE)g(\015oating)h(p)q(oin)o(t.)166 1348 y(The)d(t)o(yp)q(e)g (matc)o(hing)g(rules)g(imply)h(that)e(MPI)h(comm)o(unication)h(nev)o (er)f(en)o(tails)g(t)o(yp)q(e)g(con)o(v)o(ersion.)75 1404 y(On)20 b(the)f(other)f(hand,)i(MPI)f(requires)h(that)e(a)h (represen)o(tation)g(con)o(v)o(ersion)g(b)q(e)h(p)q(erformed)f(when)g (a)75 1461 y(t)o(yp)q(ed)i(v)m(alue)h(is)f(transferred)f(across)g(en)o (vironmen)o(ts)h(that)f(use)h(di\013eren)o(t)f(represen)o(tations)h (for)f(the)75 1517 y(datat)o(yp)q(e)g(of)g(this)h(v)m(alue.)38 b(MPI)20 b(do)q(es)h(not)f(sp)q(ecify)i(rules)g(for)d(represen)o (tation)i(con)o(v)o(ersion.)36 b(Suc)o(h)75 1573 y(con)o(v)o(ersion)21 b(is)g(exp)q(ected)i(to)d(preserv)o(e)h(in)o(teger,)h(logical)g(or)e(c) o(haracter)h(v)m(alues,)i(and)e(to)f(con)o(v)o(ert)g(a)75 1630 y(\015oating)15 b(p)q(oin)o(t)h(v)m(alue)g(to)f(the)g(nearest)g(v) m(alue)h(that)f(can)g(represen)o(ted)h(on)f(the)g(target)f(system.)166 1687 y(Ov)o(er\015o)o(w)c(and)h(under\015o)o(w)g(exceptions)h(ma)o(y)e (o)q(ccur)i(during)f(\015oating)g(p)q(oin)o(t)g(con)o(v)o(ersions;)h (con)o(v)o(er-)75 1743 y(sion)i(of)e(in)o(tegers)i(or)e(c)o(haracters)h (ma)o(y)f(also)h(lead)h(to)f(exceptions)h(when)g(a)e(v)m(alue)j(that)d (can)h(represen)o(ted)75 1800 y(in)18 b(one)g(system)f(cannot)g(b)q(e)h (represen)o(ted)g(in)h(the)e(other)g(system.)26 b(An)18 b(exception)h(o)q(ccurring)f(during)75 1856 y(represen)o(tation)f(con)o (v)o(ersion)f(results)h(in)h(a)e(failure)i(of)e(the)g(comm)o (unication;)i(an)e(error)g(o)q(ccurs)h(either)75 1912 y(in)f(the)f(send)h(op)q(eration,)f(or)g(the)g(receiv)o(e)h(op)q (eration,)f(or)g(b)q(oth.)166 1969 y(If)22 b(a)f(v)m(alue)i(sen)o(t)e (in)h(a)f(message)g(is)h(un)o(t)o(yp)q(ed)h(\(i.e.,)f(of)f(t)o(yp)q(e)h Fv(MPI)p 1370 1969 V 15 w(BYTE)p FC(\),)g(then)g(the)f(binary)75 2026 y(represen)o(tation)15 b(of)f(the)h(b)o(yte)g(stored)f(at)g(the)h (receiv)o(er)h(is)f(iden)o(tical)i(to)d(the)h(binary)h(represen)o (tation)e(of)75 2082 y(the)f(b)o(yte)g(loaded)h(at)e(the)h(sender.)20 b(This)14 b(holds)f(true,)g(whether)g(sender)h(and)f(receiv)o(er)h(run) f(in)h(the)f(same)75 2138 y(or)i(in)h(distinct)g(en)o(vironmen)o(ts;)f (no)g(represen)o(tation)h(con)o(v)o(ersion)f(is)h(required.)166 2195 y(Note)d(that)g(no)g(con)o(v)o(ersion)h(ev)o(er)f(o)q(ccurs)h (when)g(an)f(MPI)g(program)g(executes)h(in)g(a)f(homogeneous)75 2252 y(system,)i(where)h(all)g(pro)q(cesses)g(run)g(in)g(the)g(same)f (en)o(vironmen)o(t.)22 b(Also)16 b(note)f(the)h(di\013eren)o(t)f(b)q (eha)o(vior)75 2308 y(of)e Fv(MPI)p 210 2308 V 15 w(BYTE)h FC(and)f(of)g Fv(MPI)p 575 2308 V 15 w(CHAR)p FC(.)h(A)f(bu\013er)g (descriptor)g(en)o(try)g(with)g(datat)o(yp)q(e)g(of)f Fv(MPI)p 1660 2308 V 16 w(CHAR)i FC(can)75 2364 y(only)f(matc)o(h)f(a)g (C)g(v)m(ariable)i(of)e(t)o(yp)q(e)h Fo(CHAR)p FC(;)e(and)i(represen)o (tation)g(con)o(v)o(ersion)f(ma)o(y)g(o)q(ccur)h(when)g(v)m(alues)75 2421 y(of)g(t)o(yp)q(e)h Fv(MPI)p 311 2421 V 15 w(CHAR)h FC(are)e(transferred.,)g(e.g.,)f(from)h(an)g(EBCDIC)h(enco)q(ding)h(to) e(an)g(ASCI)q(I)i(enco)q(ding.)166 2478 y(Consider)h(the)f(previous)h (three)f(examples.)166 2534 y(The)c(\014rst)f(program)f(is)i(correct,)g (assuming)g(b)q(oth)f(sender)i(and)e(receiv)o(er)i(declared)f(ten)g (consecutiv)o(e)75 2591 y(real)19 b(v)m(ariables)h(in)g(storage)e (starting)g(at)g Fo(a\(1\))p FC(.)30 b(If)19 b(the)g(sender)h(and)f (receiv)o(er)g(execute)h(in)g(di\013eren)o(t)75 2647 y(en)o(vironmen)o(ts,)12 b(then)f(the)g(ten)f(real)i(v)m(alues)f(that)g (are)f(fetc)o(hed)h(from)f(the)h(send)h(bu\013er)e(will)j(b)q(e)e(con)o (v)o(erted)75 2704 y(to)g(the)h(represen)o(tation)f(for)g(reals)h(on)f (the)h(receiv)o(er)g(site)g(b)q(efore)g(they)g(are)f(stored)g(in)h(the) g(receiv)o(e)g(bu\013er.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 32 24 31 bop 75 -100 a FC(24)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FC(While)j(the)e(n)o (um)o(b)q(er)h(of)f(real)g(elemen)o(ts)h(fetc)o(hed)g(from)f(the)g (send)h(bu\013er)f(equal)i(the)e(n)o(um)o(b)q(er)h(of)f(real)75 102 y(elemen)o(ts)d(stored)e(in)i(the)f(receiv)o(e)h(bu\013er,)f(the)g (n)o(um)o(b)q(er)g(of)g(b)o(ytes)g(stored)f(need)i(not)f(equal)h(the)f (n)o(um)o(b)q(er)75 158 y(of)f(b)o(ytes)g(loaded:)19 b(e.g.)g(the)12 b(sender)h(ma)o(y)e(use)i(a)f(four)g(b)o(yte)g (represen)o(tation)h(and)f(the)g(receiv)o(er)i(an)e(eigh)o(t)75 214 y(b)o(yte)17 b(represen)o(tation)h(for)f(reals.)27 b(If)18 b(the)g(send)g(or)f(receiv)o(e)h(bu\013er)g(do)f(not)h(con)o (tain)f(ten)h(consecutiv)o(e)75 271 y(real)e(v)m(ariables,)g(then)f (the)g(program)g(is)g(erroneous,)g(and)g(its)h(b)q(eha)o(vior)f(is)h (unde\014ned.)166 331 y(The)f(second)h(program)e(is)i(erroneous,)f(and) g(its)g(b)q(eha)o(vior)h(is)g(unde\014ned.)166 390 y(The)e(third)g (program)e(is)i(correct.)19 b(The)13 b(exact)g(same)g(sequence)i(of)e (fort)o(y)f(b)o(ytes)h(that)g(w)o(ere)g(loaded)75 447 y(from)18 b(the)h(send)h(bu\013er)f(will)i(b)q(e)e(stored)g(in)h(the)f (receiv)o(e)h(bu\013er,)f(ev)o(en)h(if)f(sender)h(and)f(receiv)o(er)h (run)75 503 y(in)e(a)g(di\013eren)o(t)f(en)o(vironmen)o(t.)28 b(The)18 b(message)f(sen)o(t)g(has)h(exactly)f(the)h(same)f(length)i (\(in)f(b)o(ytes\))f(and)75 560 y(the)f(same)f(binary)h(represen)o (tation)f(as)g(the)h(message)f(receiv)o(ed.)22 b(If)16 b(the)f(v)m(ariables)i(in)f(the)g(send)g(bu\013er)75 616 y(are)f(of)h(di\013eren)o(t)g(t)o(yp)q(es)g(from)f(the)h(v)m (ariables)h(in)f(the)g(receiv)o(e)h(bu\013er,)e(or)h(they)g(are)f(of)h (the)f(same)h(t)o(yp)q(e)75 672 y(but)h(di\013eren)o(t)g(data)g (represen)o(tations)f(are)h(used,)h(then)f(the)g(bits)h(stored)e(in)i (the)f(receiv)o(e)h(bu\013er)f(ma)o(y)75 729 y(enco)q(de)f(v)m(alues)h (that)d(are)h(di\013eren)o(t)g(from)g(the)g(v)m(alues)h(they)g(enco)q (ded)g(in)g(the)g(send)f(bu\013er.)166 789 y(Data)f(represen)o(tation)i (con)o(v)o(ersion)f(also)h(applies)h(to)e(the)g(en)o(v)o(elop)q(e)i(of) e(a)g(message:)20 b(source,)15 b(des-)75 845 y(tination)h(and)f(tag)f (are)h(all)h(in)o(tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i(con)o(v) o(erted.)166 905 y(The)k(curren)o(t)g(draft)f(do)q(es)h(not)g(pro)o (vide)g(a)g(mec)o(hanism)h(for)e(in)o(ter-language)i(comm)o(unication:) 75 961 y(messages)d(sen)o(t)h(b)o(y)f(F)l(ortran)g(calls)i(should)f(b)q (e)h(receiv)o(ed)g(b)o(y)e(F)l(ortran)g(calls)h(and)g(messages)g(sen)o (t)f(b)o(y)75 1018 y(C)e(calls)g(should)h(b)q(e)g(receiv)o(ed)g(b)o(y)e (C)h(calls)h(\(this)f(follo)o(ws)f(from)h(the)f(requiremen)o(ts)i(of)e (t)o(yp)q(e)h(matc)o(hing)75 1074 y(and)g(the)h(fact)f(that)f(F)l (ortran)g(and)i(C)f(datat)o(yp)q(es)g(are)g(distinct\).)24 b(If)16 b(in)o(ter-language)h(comm)o(unication)75 1131 y(is)g(needed)i(then)e(one)g(needs)h(to)e(in)o(v)o(ok)o(e)h(C)f(comm)o (unication)i(routines)f(from)f(a)h(F)l(ortran)f(program)f(or)75 1187 y(F)l(ortran)f(comm)o(unication)i(routines)g(from)e(a)h(C)g (program.)166 1323 y Fn(Implemen)o(tati)o(on)d(note:)166 1376 y Fq(The)k(curren)o(t)i(de\014nition)e(do)q(es)h(not)f(require)h (messages)f(to)g(carry)h(data)f(t)o(yp)q(e)g(information.)22 b(A)16 b(message)75 1425 y(can)c(b)q(e)h(comp)q(osed)f(and)g(sen)o(t)h (using)f(only)g(the)h(information)c(pro)o(vided)j(b)o(y)g(the)h(send)g (call,)e(and)h(can)h(b)q(e)g(receiv)o(ed)75 1475 y(and)k(stored)g (using)g(only)f(the)h(information)d(pro)o(vided)j(b)o(y)g(the)g(receiv) o(e)h(call.)26 b(If)17 b(messages)g(are)g(sen)o(t)h(b)q(et)o(w)o(een)75 1525 y(di\013eren)o(t)c(mac)o(hines)f(then)g(one)h(can)f(either)h(use)g (a)f(\\univ)o(ersal")g(data)g(enco)q(ding)g(for)g(messages,)g(use)h (kno)o(wledge)75 1575 y(of)g(the)i(receiv)o(er)h(en)o(vironmen)o(t)d (in)g(order)i(to)f(con)o(v)o(ert)h(data)e(at)h(the)h(sender,)h(or)e (use)g(kno)o(wledge)g(of)g(the)g(sender)75 1625 y(en)o(vironmen)o(t)f (in)h(order)h(to)f(con)o(v)o(ert)i(data)e(at)g(the)h(receiv)o(er.)24 b(In)15 b(either)h(case)h(the)f(datat)o(yp)q(e)f(argumen)o(t)f(in)h (the)75 1675 y(lo)q(cal)e(call)g(can)h(b)q(e)h(used)g(to)e(deriv)o(e)i (the)f(t)o(yp)q(es)h(of)e(the)i(v)n(alues)e(transferred.)166 1728 y(Additional)f(t)o(yp)q(e)j(information)c(can)j(b)q(e)g(used)h(to) f(pro)o(vide)g(b)q(etter)h(error)g(detection.)75 1972 y Ft(3.7)59 b(Communication)18 b(Mo)r(des)75 2079 y FC(The)13 b(basic)h(send)f(op)q(eration)g(describ)q(ed)i(in)f(section)f(3.2)f (used)h(the)g Fx(standard)g FC(comm)o(unication)h(mo)q(de.)75 2136 y(In)g(suc)o(h)g(a)f(comm)o(unication)h(mo)q(de,)g(a)f(send)h(op)q (eration)g(can)f(b)q(e)h(started)f(whether)h(or)f(not)g(a)g(matc)o (hing)75 2192 y(receiv)o(e)18 b(w)o(as)e(p)q(osted.)24 b(The)17 b(completion)h(of)e(the)h(send)h(op)q(eration)e(indicates)j (that)d(the)g(message)h(and)75 2249 y(its)f(en)o(v)o(elop)q(e)h(ha)o(v) o(e)e(b)q(een)i(safely)f(stored)f(a)o(w)o(a)o(y)f(and)i(that)f(the)h (sender)g(is)g(free)g(to)f(access)h(and)f(mo)q(dify)75 2305 y(the)j(sender)h(bu\013er.)30 b(Th)o(us,)18 b(the)h(op)q(eration)f (is)h Fx(lo)q(cally-blo)r(c)o(king)p FC(:)29 b(it)19 b(do)q(es)f(not)g(return)g(un)o(til)i(the)75 2362 y(send)c(op)q (eration)f(has)g(completed)h Fx(lo)q(cally)p FC(,)i(on)d(the)g(sender)h (side.)166 2421 y(The)f(completion)i(of)e(a)g(send)h(op)q(eration)f (giv)o(es)h(no)f(indication)i(that)e(the)g(message)g(w)o(as)g(receiv)o (ed)75 2478 y(on)i(the)g(receiv)o(er)h(side.)26 b(A)17 b(blo)q(c)o(king)i(send)e(ma)o(y)g(b)q(e)g(implemen)o(ted)i(so)e(that)f (it)h(returns)g(only)h(after)e(a)75 2534 y(matc)o(hing)j(receiv)o(e)h (has)f(b)q(een)h(executed)g(on)f(the)h(receiv)o(er)f(side.)33 b(This)20 b(a)o(v)o(oids)e(the)i(need)g(to)e(bu\013er)75 2591 y(message)e(data)f(out)h(of)f(sender)i(or)e(receiv)o(er)i(memory)l (.)22 b(In)17 b(this)f(case)g(the)g(send)h(op)q(eration)f(completes)75 2647 y(only)e(after)e(the)i(matc)o(hing)f(receiv)o(e)h(has)g(started)e (executing.)21 b(On)13 b(the)h(other)f(hand,)g(it)h(is)g(also)f(p)q (ossible)75 2704 y(for)j(MPI)g(to)f(bu\013er)h(messages,)g(so)g(as)g (to)f(allo)o(w)i(the)f(sender)h(to)e(pro)q(ceed)i(ahead)g(of)e(the)i (receiv)o(er.)23 b(In)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 33 25 32 bop 75 -100 a Fs(3.7.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b FC(25)75 45 y(suc)o(h)15 b(a)g(case)g(the)g(send)g(op)q (eration)g(ma)o(y)f(complete)i(successfully)h(b)q(efore)e(the)g (message)f(w)o(as)g(receiv)o(ed.)166 116 y(There)h(are)g(t)o(w)o(o)f (additional)j(comm)o(unication)f(mo)q(des:)166 186 y(A)e(send)h(that)e (uses)h(the)g Fx(ready)g FC(comm)o(unication)g(mo)q(de)h(ma)o(y)e(b)q (e)i(started)e(only)h(if)h(the)f(matc)o(hing)75 242 y(receiv)o(e)g(is)f (already)g(p)q(osted;)g(otherwise)g(the)g(op)q(eration)g(is)g (erroneous)f(and)h(its)g(outcome)f(is)h(unde\014ned.)75 299 y(In)19 b(some)f(systems,)h(this)f(allo)o(ws)h(the)f(remo)o(v)m(al) h(of)f(a)g(hand-shak)o(e)h(op)q(eration)g(that)e(is)i(otherwise)g(re-) 75 355 y(quired,)g(and)f(results)g(in)g(impro)o(v)o(ed)g(p)q (erformance.)27 b(The)18 b(completion)h(of)e(the)h(send)g(op)q(eration) g(do)q(es)75 412 y(not)e(dep)q(end)i(on)e(the)g(status)f(of)h(a)g(matc) o(hing)g(receiv)o(e,)h(and)f(merely)h(indicates)h(that)d(the)h(send)h (bu\013er)75 468 y(can)i(b)q(e)g(reused.)30 b(A)19 b(send)g(op)q (eration)g(that)f(uses)g(the)h(ready)f(mo)q(de)h(has)g(the)f(same)g (seman)o(tics)h(as)f(a)75 525 y(standard)h(send)h(op)q(eration;)h(it)f (is)g(merely)g(that)f(it)g(can)h(b)q(e)g(used)g(only)g(in)g(some)f(of)g (the)h(situations)75 581 y(where)15 b(a)g(regular)h(sen)o(t)f(can)g(b)q (e)h(used.)166 652 y(A)i(send)g(that)f(uses)g(the)h Fx(sync)o(hronous)e FC(mo)q(de)i(can)g(b)q(e)g(started)f(whether)g(or)g(not)h(a)f(matc)o (hing)75 708 y(receiv)o(e)12 b(w)o(as)e(p)q(osted.)19 b(Ho)o(w)o(ev)o(er,)10 b(the)h(send)h(will)g(complete)g(successfully)h (only)f(if)f(a)g(matc)o(hing)g(receiv)o(e)g(is)75 765 y(p)q(osted,)h(and)g(the)g(receiv)o(e)g(op)q(eration)g(has)f(started)g (to)g(receiv)o(e)i(the)e(message)g(sen)o(t)h(b)o(y)f(the)h(sync)o (hronous)75 821 y(send.)29 b(\(I.e.,)19 b(the)f(receiv)o(e)h(has)f(b)q (een)h(p)q(osted,)g(and)g(the)f(incoming)h(message)f(has)g(b)q(een)i (matc)o(hed)e(to)75 878 y(this)g(p)q(osted)g(receiv)o(e.\))28 b(Th)o(us,)18 b(the)g(completion)g(of)g(a)f(sync)o(hronous)h(send)g (not)f(only)i(indicates)g(that)75 934 y(the)g(send)h(bu\013er)f(can)g (b)q(e)h(reused,)g(but)f(also)g(indicates)i(that)d(the)i(receiv)o(er)f (has)g(reac)o(hed)h(a)f(certain)75 990 y(p)q(oin)o(t)e(in)h(its)f (execution,)h(namely)g(that)e(it)h(has)g(started)f(executing)i(the)f (matc)o(hing)g(receiv)o(e.)26 b(If)17 b(b)q(oth)75 1047 y(sends)f(and)h(receiv)o(es)f(are)g(blo)q(c)o(king)h(op)q(erations)f (then)h(the)f(use)g(of)f(the)h(sync)o(hronous)g(mo)q(de)g(pro)o(vides) 75 1103 y(sync)o(hronous)h(comm)o(unication)g(seman)o(tics:)23 b(a)16 b(comm)o(unication)h(do)q(es)g(not)g(complete)g(at)f(either)h (end)75 1160 y(b)q(efore)12 b(b)q(oth)g(pro)q(cesses)g(\\attend")f(to)g (the)h(comm)o(unication;)h(the)f(completion)h(of)e(a)h(sync)o(hronous)f (send)75 1216 y(is)16 b(a)f Fx(global)i FC(ev)o(en)o(t,)d(and)h(the)h (op)q(eration)f(is)h Fx(globally-blo)q(cking)p FC(.)166 1287 y(Tw)o(o)22 b(additional)j(send)e(functions)h(are)f(pro)o(vided)h (for)f(the)g(t)o(w)o(o)f(additional)i(comm)o(unication)75 1343 y(mo)q(des.)32 b(The)19 b(comm)o(unication)h(mo)q(de)f(is)h (indicated)g(b)o(y)f(a)g(one)g(letter)h(pre\014x:)28 b Fo(R)18 b FC(for)h(ready)g(and)g Fo(S)75 1400 y FC(for)c(sync)o (hronous.)166 1470 y(Send)h(in)g(ready)f(mo)q(de)75 1588 y Fv(MPI)p 160 1588 14 2 v 16 w(RSEND)h(\(buf,)f(count,)h(datat)o(yp)q (e,)h(dest,)f(tag,)f(comm\))117 1679 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1782 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1885 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1988 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2092 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 2195 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))75 2333 y Fo(int)23 b(MPI)p 245 2333 15 2 v 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2333 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2390 y(int)h(tag,)f(MPI)p 683 2390 V 17 w(Comm)g(comm\))75 2490 y(MPI)p 150 2490 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 2547 y()f(BUF\(*\))170 2603 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 2704 y FC(Send)16 b(in)g(sync)o(hronous)f(mo)q(de)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 34 26 33 bop 75 -100 a FC(26)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SSEND)g(\(buf,)f(count,)h(datat)o(yp)q(e,)g(dest,)h (tag,)e(comm\))117 123 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 199 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 276 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h (bu\013er)g(elemen)o(t)e(\(handle\))117 352 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o(teger\))117 429 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 505 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))75 630 y Fo(int)23 b(MPI)p 245 630 15 2 v 17 w(Ssend\(void*)g(buf,)g(int)g (count,)g(MPI)p 1002 630 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 687 y(int)h(tag,)f(MPI)p 683 687 V 17 w(Comm)g(comm\))75 774 y(MPI)p 150 774 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 830 y()f(BUF\(*\))170 887 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 974 y FC(There)16 b(is)g(only)g(one)g(receiv)o(e)h(mo)q(de,)e(whic)o(h) i(can)f(matc)o(h)f(an)o(y)g(of)h(the)f(send)i(mo)q(des.)k(The)16 b(receiv)o(e)75 1030 y(op)q(eration)g(describ)q(ed)h(in)g(the)e(last)h (section)g(is)g Fx(blo)q(c)o(king)p FC(:)22 b(it)16 b(returns)f(only)h (after)f(the)h(receiv)o(e)g(bu\013er)75 1087 y(con)o(tains)h(the)h (newly)g(receiv)o(ed)g(message.)26 b(It)17 b(is)h Fx(async)o(hronous)p FC(:)23 b(the)17 b(completion)i(of)d(a)h(receiv)o(e)h(is)75 1143 y(a)e(lo)q(cal)i(op)q(eration,)f(and)f(a)g(receiv)o(e)i(can)f (complete)g(b)q(efore)g(the)f(matc)o(hing)h(send)g(has)f(completed)i (\(of)75 1199 y(course,)d(it)g(can)h(complete)g(only)f(after)g(the)g (matc)o(hing)g(send)h(has)f(started\).)166 1257 y(Comm)o(unication)e (imp)q(oses)g(an)f(order)h(on)f(the)h(ev)o(en)o(ts)f(o)q(ccurring)h(at) f(the)h(comm)o(unicating)g(no)q(des.)75 1313 y(It)i(is)g(alw)o(a)o(ys)e (the)i(case)g(that)e(the)i(completion)h(of)e(a)g(receiv)o(e)i(o)q (ccurs)e(after)g(the)h(start)e(of)h(the)h(matc)o(hing)75 1369 y(send.)29 b(If)19 b(the)f(sync)o(hronous)g(send)h(mo)q(de)f(is)h (used,)g(then)g(it)f(is)h(also)f(the)g(case)g(that)g(the)g(completion) 75 1426 y(of)i(a)h(send)g(o)q(ccurs)g(after)f(the)g(start)g(of)g(the)h (matc)o(hing)f(receiv)o(e.)38 b(Of)20 b(course,)i(on)f(eac)o(h)f(pro)q (cess,)i(a)75 1482 y(comm)o(unication)14 b(completes)h(after)d(it)i(is) g(started.)19 b(No)13 b(other)h(order)f(is)h(imp)q(osed)h(b)o(y)e(MPI.) g(E.g.,)g(if)h(the)75 1539 y(standard)h(send)g(mo)q(de)h(is)f(used,)h (the)f(send)h(op)q(eration)f(ma)o(y)f(complete)i(b)q(efore)g(the)f (matc)o(hing)g(receiv)o(e)75 1595 y(w)o(as)f(started.)166 1652 y(In)h(a)f(m)o(ulti-threaded)i(implemen)o(tation)g(of)e(MPI,)h (the)f(system)g(ma)o(y)g(desc)o(hedule)j(a)d(thread)h(that)75 1709 y(is)21 b(blo)q(c)o(k)o(ed)g(on)g(a)f(send)h(or)e(receiv)o(e)j(op) q(eration,)f(and)g(sc)o(hedule)h(another)e(thread)g(for)g(execution)h (in)75 1765 y(the)d(same)g(address)g(space.)28 b(In)19 b(suc)o(h)f(a)f(case)h(it)h(the)f(user)g(resp)q(onsibilit)o(y)i(not)e (to)f(access)h(or)f(mo)q(dify)75 1822 y(a)f(comm)o(unication)h (bu\013er)f(un)o(til)i(the)e(comm)o(unication)h(completes;)g(otherwise) g(the)f(outcome)g(of)g(the)75 1878 y(computation)f(is)h(unde\014ned.) 166 2011 y Fn(Implemen)o(tati)o(on)10 b(note:)34 b Fq(A)13 b(ready)f(send)h(can)f(b)q(e)h(implemen)o(ted)d(as)i(a)g(standard)h (send;)g(in)e(suc)o(h)i(a)f(case)75 2061 y(there)j(will)e(b)q(e)h(no)g (p)q(erformance)g(adv)n(an)o(tage)f(\(or)h(disadv)n(an)o(tage\))f(for)g (the)i(use)f(of)g(ready)g(send.)166 2112 y(A)f(standard)h(send)h(can)e (b)q(e)h(implemen)o(ted)e(as)h(a)g(sync)o(hronous)i(send.)j(In)c(suc)o (h)g(a)f(case,)h(no)f(data)h(bu\013ering)75 2161 y(is)g(needed.)166 2212 y(A)g(p)q(ossible)g(comm)o(unication)c(proto)q(col)k(for)g(the)g (v)n(arious)f(comm)o(unication)e(mo)q(des)i(is)h(outlined)f(b)q(elo)o (w:)166 2262 y Fk(ready)21 b(send)p Fq(:)c(The)d(message)g(is)f(sen)o (t)i(as)f(so)q(on)g(as)g(p)q(ossible.)166 2313 y Fk(synchronous)19 b(send:)29 b Fq(The)20 b(sender)g(sends)h(a)e(request-to-send)j (message.)34 b(The)19 b(receiv)o(er)i(stores)g(this)75 2363 y(request.)f(When)15 b(a)f(matc)o(hing)e(receiv)o(e)j(is)f(p)q (osted,)h(the)g(receiv)o(er)h(sends)f(bac)o(k)f(a)g(p)q (ermission-to-send)g(message,)75 2412 y(and)g(the)g(sender)i(no)o(w)d (sends)i(the)g(message.)166 2463 y Fk(standard)20 b(send:)e Fq(First)d(proto)q(col)f(ma)o(y)e(b)q(e)j(used)h(for)d(short)i (messages,)f(and)g(second)i(proto)q(col)e(for)g(long)75 2513 y(messages.)166 2563 y(Additional)h(con)o(trol)h(messages)h(migh)o (t)e(b)q(e)i(needed)h(for)e(\015o)o(w)g(con)o(trol)h(and)f(error)i (reco)o(v)o(ery)m(.)27 b(Of)16 b(course,)75 2613 y(there)f(are)f(man)o (y)f(other)h(p)q(ossible)g(c)o(hoices.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 35 27 34 bop 75 -100 a Fs(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FC(27)75 45 y Ft(3.8)59 b(Nonblo)r(cking)20 b(communication)75 147 y FC(One)e(can)g(impro)o(v)o(e)f(p)q(erformance)g(on)h(man)o(y)e (systems)h(b)o(y)h(o)o(v)o(erlapping)f(comm)o(unication)i(and)e(com-)75 203 y(putation.)35 b(This)21 b(is)g(esp)q(ecially)i(true)d(on)g (systems)g(where)g(comm)o(unication)h(can)g(b)q(e)g(executed)g(au-)75 259 y(tonomously)12 b(b)o(y)h(an)g(in)o(telligen)o(t)h(comm)o (unication)g(con)o(troller.)19 b(Ligh)o(t-w)o(eigh)o(t)13 b(threads)f(are)h(one)g(mec)o(h-)75 316 y(anism)21 b(for)g(ac)o (hieving)h(suc)o(h)g(o)o(v)o(erlap.)37 b(An)21 b(alternativ)o(e)h(mec)o (hanism)g(that)e(often)h(leads)g(to)g(b)q(etter)75 372 y(p)q(erformance)e(is)g(to)f(use)h Fx(non)o(blo)q(c)o(king)j(comm)o (unication)p FC(.)32 b(A)19 b(non)o(blo)q(c)o(king)h(send)f(call)h (initiates)75 429 y(the)15 b(send)h(op)q(eration,)f(but)h(do)q(es)f (not)g(complete)h(it.)k(The)15 b(send)h(call)h(will)f(return)g(b)q (efore)f(the)g(message)75 485 y(w)o(as)d(copied)h(out)f(of)g(the)g (send)h(bu\013er.)19 b(A)13 b(separate)e(call)j(is)f(needed)g(to)f (complete)h(the)g(comm)o(unication,)75 542 y(i.e.)20 b(to)13 b(v)o(erify)i(that)e(the)h(data)g(has)g(b)q(een)h(copied)g(out) f(of)g(the)g(send)h(bu\013er.)k(With)c(suitable)g(hardw)o(are,)75 598 y(the)k(transfer)f(of)h(data)f(out)h(of)g(the)g(sender)g(memory)g (ma)o(y)f(pro)q(ceed)i(concurren)o(tly)g(with)f(computa-)75 655 y(tions)g(done)g(at)f(the)g(sender)h(after)f(the)h(send)g(w)o(as)f (initiated)i(and)e(b)q(efore)h(it)g(completed.)31 b(Similarly)l(,)75 711 y(a)19 b(non)o(blo)q(c)o(king)h(receiv)o(e)g(initiates)h(the)e (receiv)o(e)h(op)q(eration,)g(but)f(do)q(es)g(not)g(complete)h(it.)32 b(The)19 b(call)75 768 y(will)f(return)f(b)q(efore)g(a)f(message)g(is)i (stored)e(in)o(to)g(the)h(receiv)o(e)h(bu\013er.)24 b(A)17 b(separate)f(call)h(is)h(needed)g(to)75 824 y(complete)f(the)f(receiv)o (e)i(op)q(eration)e(and)g(v)o(erify)h(that)e(the)i(data)e(has)i(b)q (een)g(receiv)o(ed)g(in)o(to)g(the)f(receiv)o(e)75 880 y(bu\013er.)k(With)14 b(suitable)h(hardw)o(are,)f(the)g(transfer)f(of)h (data)g(in)o(to)g(the)g(receiv)o(er)h(memory)f(ma)o(y)f(pro)q(ceed)75 937 y(concurren)o(tly)f(with)g(computations)f(done)g(after)g(the)g (receiv)o(e)h(w)o(as)f(initiated)i(and)e(b)q(efore)h(it)f(completed.)75 993 y(The)17 b(use)h(of)f(non)o(blo)q(c)o(king)h(receiv)o(es)g(ma)o(y)f (also)g(a)o(v)o(oid)g(system)f(bu\013ering)i(and)f(memory)g(to)g (memory)75 1050 y(cop)o(ying,)e(as)g(information)g(is)h(pro)o(vided)g (early)g(on)f(the)g(lo)q(cation)h(of)f(the)g(receiv)o(e)h(bu\013er.)166 1106 y(Non)o(blo)q(c)o(king)g(sends)g(can)g(use)f(the)h(same)f(three)g (mo)q(des)h(as)f(blo)q(c)o(king)h(sends:)21 b Fo(standard)p FC(,)14 b Fo(ready)75 1163 y FC(and)21 b Fo(synchronous)p FC(.)35 b(These)21 b(carry)g(the)g(same)f(meaning.)37 b(The)21 b(initiation)i(and)e(completion)h(of)e(a)75 1219 y(standard)e(send)i(do)f(not)f(dep)q(end)j(on)d(the)h(status)f(of) h(a)f(matc)o(hing)h(receiv)o(e.)32 b(A)19 b(ready)g(send)g(can)g(b)q(e) 75 1276 y(initiated)12 b(only)f(if)g(a)g(matc)o(hing)f(receiv)o(e)i (has)e(already)h(b)q(een)h(initiated,)g(otherwise)f(the)g(call)g(is)g (erroneous;)75 1332 y(its)18 b(completion)g(do)q(es)g(not)f(dep)q(end)i (on)f(the)f(status)g(of)g(a)g(matc)o(hing)h(receiv)o(e.)27 b(A)18 b(sync)o(hronous)f(send)75 1389 y(can)f(b)q(e)h(initiated)h(b)q (efore)f(a)f(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)i(initiated,)g (but)e(will)i(complete)f(only)g(after)75 1445 y(a)f(matc)o(hing)f (receiv)o(e)i(has)f(b)q(een)h(initiated,)g(and)f(has)g(started)f (receiving)j(the)e(message)f(generated)h(b)o(y)75 1501 y(the)f(send)h(op)q(eration.)166 1558 y(In)j(all)h(cases)e(the)h(op)q (eration)g(that)e(initiates)j(the)f(comm)o(unication)g(\(send)g(or)f (receiv)o(e\))h(is)g(a)f(lo-)75 1614 y(cal,)23 b(non)o(blo)q(c)o(king)g (call.)39 b(Suc)o(h)22 b(a)f(call)h(ma)o(y)f(fail)h(b)q(ecause)g(the)g (system)e(has)i(exhausted)f(a)o(v)m(ailable)75 1671 y(resources)14 b(\(e.g.,)f(exceeded)j(the)f(limit)h(on)e(n)o(um)o(b)q(er)h(of)f(p)q (ending)i(comm)o(unications)f(p)q(er)g(no)q(de\).)20 b(Go)q(o)q(d)75 1727 y(qualit)o(y)h(implemen)o(tations)g(of)e(MPI)h (will)i(set)e(these)g(limits)h(high)g(enough)f(so)g(that)f(a)h(non)o (blo)q(c)o(king)75 1784 y(comm)o(unication)c(initiation)h(call)f(will)h (fail)f(only)g(in)g(\\pathological")f(cases.)166 1840 y(Non)o(blo)q(c)o(king)h(sends)g(can)f(b)q(e)h(matc)o(hed)f(with)h(blo) q(c)o(king)g(receiv)o(es,)g(and)f(vice-v)o(ersa.)75 1960 y Fp(3.8.1)49 b(Communication)17 b(Objects)75 2046 y FC(Non)o(blo)q(c)o(king)i(comm)o(unications)f(use)g(opaque)g(comm)o (unication)g(request)g(ob)s(jects)f(to)g(iden)o(tify)i(com-)75 2103 y(m)o(unication)d(op)q(erations)f(and)h(matc)o(h)e(the)i(op)q (eration)f(that)g(initiates)h(the)f(comm)o(unication)h(with)g(the)75 2159 y(op)q(eration)h(that)f(terminates)h(it.)26 b(These)17 b(are)g(system)f(ob)s(jects)g(that)h(are)f(accessed)i(via)f(a)g (handle.)26 b(A)75 2215 y(request)13 b(ob)s(ject)g(iden)o(ti\014es)i(v) m(arious)f(prop)q(erties)g(of)f(a)g(comm)o(unication)h(op)q(eration,)g (suc)o(h)f(as)g(the)h(\(send)75 2272 y(or)h(receiv)o(e\))h(bu\013er)f (that)f(is)i(asso)q(ciated)g(with)f(it,)h(its)f(con)o(text,)f(the)i (tag)e(and)i(destination)g(argumen)o(ts)75 2328 y(to)d(b)q(e)h(used)f (for)g(a)g(send,)h(or)e(the)i(tag)e(and)i(source)f(argumen)o(ts)f(to)h (b)q(e)h(used)g(for)e(a)h(receiv)o(e.)20 b(In)14 b(addition,)75 2385 y(this)20 b(ob)s(ject)f(stores)g(information)g(ab)q(out)g(the)h (status)f(of)g(the)g(p)q(ending)j(comm)o(unication)e(op)q(eration)75 2441 y(that)14 b(is)i(p)q(erformed)g(with)f(this)h(ob)s(ject.)75 2561 y Fp(3.8.2)49 b(Communication)17 b(initiation)75 2647 y FC(W)l(e)h(use)g(the)g(same)g(naming)g(con)o(v)o(en)o(tions)g (as)f(for)g(blo)q(c)o(king)j(comm)o(unication:)26 b(a)17 b(pre\014x)h(of)g Fo(R)f FC(\()p Fo(S)p FC(\))g(is)75 2704 y(used)h(for)f Fo(READY)g FC(\()p Fo(SYNCHRONOUS)p FC(\))f(mo)q(de.)27 b(In)19 b(addition)f(a)g(pre\014x)g(of)f Fo(I)h FC(\(for)e Fo(IMMEDIATE)p FC(\))h(indicates)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 36 28 35 bop 75 -100 a FC(28)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FC(that)e(the)i (call)g(is)g(non)o(blo)q(c)o(king.)166 113 y(Initiate)g(a)f(standard)g (mo)q(de)g(non)o(blo)q(c)o(king)i(send.)75 228 y Fv(MPI)p 160 228 14 2 v 16 w(ISEND\(buf,)e(count,)h(datat)o(yp)q(e,)h(dest,)f (tag,)f(comm,)e(request\))117 317 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 415 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 514 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h (bu\013er)g(elemen)o(t)e(\(handle\))117 612 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o(teger\))117 710 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 809 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 907 y(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 1043 y Fo(int)23 b(MPI)p 245 1043 15 2 v 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1043 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1099 y(int)h(tag,)f(MPI)p 683 1099 V 17 w(Comm)g(comm,)g(MPI)p 1034 1099 V 17 w(Comm)p 1147 1099 V 17 w(request)g(request\))75 1197 y(MPI)p 150 1197 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(REQUEST,)e(IERROR\))170 1254 y()h(BUF\(*\))170 1310 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1408 y FC(Initiate)16 b(a)f(ready)g(mo)q(de)h(non)o(blo)q (c)o(king)g(send.)75 1524 y Fv(MPI)p 160 1524 14 2 v 16 w(IRSEND\(buf,)g(count,)g(datat)o(yp)q(e,)g(dest,)g(tag,)f(comm,)e (request\))117 1612 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1711 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1809 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1907 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2006 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 2104 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 2202 y(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 2338 y Fo(int)23 b(MPI)p 245 2338 15 2 v 17 w(Irsend\(void*)f(buf,)i (int)f(count,)g(MPI)p 1026 2338 V 17 w(Datatype)g(datatype,)f(int)i (dest,)393 2395 y(int)g(tag,)f(MPI)p 683 2395 V 17 w(Comm)g(comm,)g (MPI)p 1034 2395 V 17 w(Comm)p 1147 2395 V 17 w(request)g(request\))75 2493 y(MPI)p 150 2493 V 17 w(IRSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,) g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2549 y()g(BUF\(*\))170 2606 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 2704 y FC(Initiate)16 b(a)f(sync)o(hronous)g(mo)q(de)h(non) o(blo)q(c)o(king)g(send.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 37 29 36 bop 75 -100 a Fs(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FC(29)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ISSEND\(buf,)15 b(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm,)e(request\))117 127 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 212 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 297 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 382 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o(teger\))117 467 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 551 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 636 y(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 766 y Fo(int)23 b(MPI)p 245 766 15 2 v 17 w(Issend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 766 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 822 y(int)g(tag,)f(MPI)p 683 822 V 17 w(Comm)g(comm,)g(MPI)p 1034 822 V 17 w(Comm)p 1147 822 V 17 w(request)g(request\))75 913 y(MPI)p 150 913 V 17 w(ISSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 970 y()g(BUF\(*\))170 1026 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1118 y FC(Initiate)16 b(a)f(non)o(blo)q(c)o(king)i(receiv)o (e.)75 1226 y Fv(MPI)p 160 1226 14 2 v 16 w(IRECV)e(\(buf,)h(count,)g (datat)o(yp)q(e,)g(source,)g(tag,)e(comm,)f(request\))117 1308 y Fq(OUT)108 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 1393 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1478 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(handle\))117 1563 y(IN)155 b Fv(source)468 b Fq(rank)14 b(of)f(source)j(\(in)o(teger\))117 1648 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 1733 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 1817 y(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 1947 y Fo(int)23 b(MPI)p 245 1947 15 2 v 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1947 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 2003 y(int)h(tag,)f(MPI)p 683 2003 V 17 w(Comm)g(comm,)g(MPI)p 1034 2003 V 17 w(Comm)p 1147 2003 V 17 w(request)g(request\))75 2094 y(MPI)p 150 2094 V 17 w(IRECV\(BUF,)f(COUNT,)h(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2151 y()g(BUF\(*\))170 2207 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 2299 y FC(These)c(calls)g(allo)q (cate)h(a)e(comm)o(unication)h(request)f(and)h(asso)q(ciate)g(it)f (with)h(the)g(handle.)31 b(The)75 2355 y(request)11 b(can)f(b)q(e)i (used)f(later)g(to)f(query)g(the)h(status)f(of)g(the)h(comm)o (unication)g(or)f(w)o(ait)g(for)g(its)h(completion.)166 2416 y(A)17 b(non)o(blo)q(c)o(king)i(send)f(call)g(indicates)h(that)e (the)g(system)g(ma)o(y)g(start)f(cop)o(ying)h(data)g(out)g(of)g(the)75 2473 y(send)g(bu\013er.)24 b(The)17 b(sender)g(should)g(not)g(access)f (an)o(y)g(part)g(of)g(the)h(send)g(bu\013er)g(after)e(a)i(non)o(blo)q (c)o(king)75 2529 y(send)f(op)q(eration)f(returns,)g(un)o(til)h(the)g (send)f(completes.)166 2591 y(A)f(non)o(blo)q(c)o(king)h(receiv)o(e)g (call)h(indicates)f(that)f(the)g(sytem)f(ma)o(y)h(start)f(writing)h (data)g(in)o(to)g(the)g(re-)75 2647 y(ceiv)o(e)e(bu\013er.)19 b(The)11 b(receiv)o(er)h(should)h(not)e(access)g(an)o(y)g(part)g(of)g (the)g(receiv)o(e)i(bu\013er)e(after)g(a)g(non)o(blo)q(c)o(king)75 2704 y(receiv)o(e)16 b(op)q(eration)g(returns,)e(un)o(til)j(the)e (receiv)o(e)h(completes.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 38 30 37 bop 75 -100 a FC(30)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(3.8.3)49 b(Communication)17 b(Completion)75 134 y FC(The)g(functions)g Fv(MPI)p 452 134 14 2 v 16 w(W)l(AIT)f FC(and)g Fv(MPI)p 768 134 V 16 w(TEST)h FC(are)f(used)h(to)f(complete)h(a)f(non)o(blo)q (c)o(king)h(comm)o(unica-)75 191 y(tion.)i(The)13 b(completion)h(of)e (a)g(send)h(op)q(eration)g(indicates)h(that)e(the)g(sender)h(is)g(no)o (w)f(free)h(to)f(up)q(date)h(the)75 247 y(lo)q(cations)k(in)f(the)h (send)f(bu\013er)g(\(the)g(send)g(op)q(eration)g(itself)h(lea)o(v)o(es) f(the)g(con)o(ten)o(t)g(of)f(the)h(send)h(bu\013er)75 304 y(unc)o(hanged\).)25 b(It)16 b(do)q(es)h(not)g(indicate)h(that)e (the)h(message)f(has)g(b)q(een)i(receiv)o(ed;)g(rather,)e(it)h(ma)o(y)f (ha)o(v)o(e)75 360 y(b)q(een)e(bu\013ered)f(b)o(y)f(the)h(comm)o (unication)g(subsystem.)19 b(Ho)o(w)o(ev)o(er,)12 b(if)h(a)f Fo(synchronous)f FC(mo)q(de)i(send)g(w)o(as)75 417 y(used,)i(the)g (completion)h(of)f(the)f(send)i(op)q(eration)f(indicates)h(that)e(a)h (matc)o(hing)g(receiv)o(e)h(w)o(as)e(initiated,)75 473 y(and)h(that)g(the)g(message)g(will)i(ev)o(en)o(tually)f(b)q(e)g (receiv)o(ed)g(b)o(y)f(this)h(matc)o(hing)f(receiv)o(e.)166 531 y(The)20 b(completion)h(of)e(a)g(receiv)o(e)i(op)q(eration)f (indicates)h(that)e(the)h(receiv)o(e)g(bu\013er)g(con)o(tains)g(the)75 588 y(receiv)o(ed)f(message,)f(and)g(that)f(the)h(status)f(ob)s(ject)h (is)g(set;)h(the)f(receiv)o(er)h(is)f(no)o(w)f(free)h(to)g(access)g (the)75 644 y(receiv)o(e)g(bu\013er.)27 b(It)17 b(do)q(es)h(not)f (indicate)i(that)d(the)i(matc)o(hing)f(send)h(op)q(eration)g(has)f (completed)h(\(but)75 701 y(indicates,)e(of)f(course,)g(that)f(the)i (send)f(w)o(as)g(initiated\).)75 806 y Fv(MPI)p 160 806 V 16 w(W)l(AIT\(request,)h(status\))117 885 y Fq(INOUT)62 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))117 964 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\)) 75 1090 y Fo(int)23 b(MPI)p 245 1090 15 2 v 17 w(Wait\(MPI)p 454 1090 V 16 w(Comm)p 566 1090 V 17 w(request)g(*request,)f(MPI)p 1084 1090 V 17 w(Status)h(*status\))75 1178 y(MPI)p 150 1178 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 1235 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1235 V 16 w(STATUS)p 979 1235 V 16 w(SIZE\),)g(IERROR)166 1323 y FC(A)14 b(call)h(to)f Fv(MPI)p 436 1323 14 2 v 15 w(W)l(AIT)h FC(returns)e(when)i(the)f(send) h(op)q(eration)f(iden)o(ti\014ed)i(b)o(y)e Fv(request)i FC(is)f(complete.)75 1380 y(If)21 b(the)g(comm)o(unication)h(ob)s(ject) f(asso)q(ciated)g(with)g(this)h(request)f(w)o(as)f(created)h(b)o(y)g(a) g(non)o(blo)q(c)o(king)75 1436 y(send)13 b(or)g(receiv)o(e)h(call,)g (then)f(the)g(ob)s(ject)f(is)h(deallo)q(cated)i(b)o(y)d(the)h(call)h (to)25 b Fv(MPI)p 1431 1436 V 16 w(W)l(AIT)13 b FC(and)g(the)g(request) 75 1493 y(b)q(ecomes)j(n)o(ull.)166 1551 y(The)h(call)g(returns)g(in)g Fv(status)h FC(information)f(on)f(the)h(completed)g(op)q(eration.)24 b(The)17 b(status)f(ob)s(ject)75 1607 y(for)g(a)h(receiv)o(e)h(op)q (eration)g(can)f(b)q(e)h(queried)g(using)g(the)f(functions)h(describ)q (ed)h(in)f(section)g(3.3.1.)24 b(The)75 1664 y(status)16 b(argumen)o(t)g(is)h(not)g(used)g(or)g(up)q(dated)g(for)g(a)f(send)h (op)q(eration,)h(and)f(an)f(arbitrary)h(v)m(alue)h(ma)o(y)75 1720 y(b)q(e)e(passed)f(for)g(this)h(argumen)o(t)e(in)i(the)f(call.)75 1826 y Fv(MPI)p 160 1826 V 16 w(TEST\(request,)h(\015ag,)f(status\))117 1905 y Fq(INOUT)62 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))117 1983 y(OUT)108 b Fv(\015ag)518 b Fq(\(logical\))117 2062 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\))75 2188 y Fo(int)23 b(MPI)p 245 2188 15 2 v 17 w(Test\(MPI)p 454 2188 V 16 w(Comm)p 566 2188 V 17 w(request)g(*request,)f(int)i(flag,)f(MPI)p 1323 2188 V 17 w(Status)g(*status\))75 2277 y(MPI)p 150 2277 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 2333 y(LOGICAL)g(FLAG)170 2390 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 2390 V 16 w(STATUS)p 979 2390 V 16 w(SIZE\),)g(IERROR)166 2478 y FC(A)17 b(call)i(to)e Fv(MPI)p 446 2478 14 2 v 15 w(TEST)h FC(returns)f Fv(\015ag)h(=)f(true)h FC(if)g(the)g(op)q (eration)f(iden)o(ti\014ed)j(b)o(y)d Fv(request)i FC(is)f(com-)75 2534 y(plete.)37 b(In)21 b(suc)o(h)g(a)f(case,)i(the)e(status)g(ob)s (ject)g(is)h(set)f(to)g(con)o(tain)h(information)g(on)f(the)h (completed)75 2591 y(op)q(eration;)15 b(if)h(the)f(comm)o(unication)h (ob)s(ject)e(w)o(as)h(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)h (send)g(or)f(receiv)o(e,)h(then)75 2647 y(it)i(is)g(deallo)q(cated)i (and)e(the)g(request)f(b)q(ecomes)i(n)o(ull.)29 b(The)18 b(call)h(returns)f Fv(\015ag=false)p FC(,)g(otherwise.)28 b(In)75 2704 y(suc)o(h)16 b(a)e(case,)h(the)h(v)m(alue)g(of)f(the)g (status)f(ob)s(ject)h(is)h(unde\014ned.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 39 31 38 bop 75 -100 a Fs(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FC(31)166 45 y Fv(MPI)p 251 45 14 2 v 16 w(TEST)15 b FC(is)h(a)f(lo)q(cal,)h(non)o(blo)q(c)o(king)g(op)q(eration.)166 102 y(The)11 b(return)g(status)f(ob)s(ject)h(for)f(a)h(receiv)o(e)h(op) q(eration)f(carries)g(information)g(that)g(can)g(b)q(e)g(accessed)75 158 y(using)16 b(the)e(functions)i(describ)q(ed)h(in)e(section)h (3.3.1.)i(The)d(status)f(argumen)o(t)g(is)h(not)f(used)i(or)e(up)q (dated)75 214 y(for)h(a)f(send)i(op)q(eration)g(and)f(an)g(arbitrary)g (v)m(alue)h(can)f(b)q(e)h(passed)g(for)e(this)i(argumen)o(t)e(in)i(the) g(call.)166 271 y(In)f(a)f(m)o(ulti-threaded)i(en)o(vironmen)o(t,)e (the)g(use)h(of)f(a)g(blo)q(c)o(king)i(receiv)o(e)f(op)q(eration)g(\()f Fv(MPI)p 1732 271 V 15 w(W)l(AIT)p FC(\))75 327 y(ma)o(y)k(allo)o(w)i (the)f(op)q(erating)g(system)g(to)g(de-sc)o(hedule)i(the)e(blo)q(c)o(k) o(ed)h(thread)f(and)h(sc)o(hedule)g(another)75 384 y(thread)j(for)g (execution,)j(if)e(suc)o(h)g(is)f(a)o(v)m(ailable.)47 b(The)23 b(use)h(of)f(a)g(non)o(blo)q(c)o(king)h(receiv)o(e)h(op)q (eration)75 440 y(\()p Fv(MPI)p 178 440 V 15 w(TEST)p FC(\))e(allo)o(ws)f(the)g(user)g(to)g(sc)o(hedule)i(alternativ)o(e)e (activities)i(within)f(a)f(single)h(thread)f(of)75 497 y(execution.)166 553 y(Example:)75 628 y Fo(CALL)h (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 685 y(IF\(rank.EQ.0\))123 741 y(THEN)170 797 y(CALL)h(MPI_ISEND\(request,)d(a\(1\),)i(10,)h (MPI_REAL,)e(1,)i(tag,)f(comm,)h(ierr\))170 854 y(****)g(do)f(some)h (computation)e(to)i(mask)f(latency)g(****)170 910 y(CALL)h (MPI_WAIT\(request,)d(status,)i(ierr\))123 967 y(ELSE)170 1023 y(CALL)h(MPI_IRECV\(request,)d(a\(1\),)i(15,)h(MPI_REAL,)e(0,)i (tag,)f(comm,)h(ierr\))170 1080 y(****)g(do)f(some)h(computation)e(to)i (mask)f(latency)g(****)170 1136 y(CALL)h(MPI_WAIT\(request,)d(status,)i (ierr\))75 1193 y(END)g(IF)166 1267 y FC(The)f(functions)h Fv(MPI)p 554 1267 V 15 w(W)l(AIT)f FC(and)g Fv(MPI)p 881 1267 V 16 w(TEST)g FC(can)g(b)q(e)g(used)h(to)e(complete)h(b)q(oth) g(sends)h(and)75 1324 y(receiv)o(es;)14 b(they)f(are)f(also)h(used)g (to)f(complete)h(an)o(y)g(other)f(non)o(blo)q(c)o(king)i(comm)o (unication)f(call)h(pro)o(vided)75 1380 y(b)o(y)h(MPI.)75 1499 y Fp(3.8.4)49 b(Semantics)15 b(of)i(Nonblo)q(cking)h (Communications)75 1584 y FC(The)11 b(seman)o(tics)f(of)h(non)o(blo)q (c)o(king)g(comm)o(unication)h(is)f(de\014ned)h(b)o(y)e(suitably)i (extending)g(the)e(de\014nitions)75 1641 y(in)16 b(section)g(3.4)e(for) h(order)g(and)g(progress.)166 1697 y(Non)o(blo)q(c)o(king)k(comm)o (unication)f(op)q(erations)g(are)g(ordered)g(according)g(to)f(the)h (execution)h(order)75 1754 y(of)h(the)g(calls)h(that)e(initiate)j(the)e (comm)o(unication;)i(this)f(de\014nes)g(the)f(meaning)h(of)e(non-o)o(v) o(ertaking)75 1810 y(comm)o(unication.)166 1867 y(Example:)75 1946 y Fo(CALL)k(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2003 y(IF)h(\(RANK.EQ.0\))147 2059 y(THEN)218 2116 y(CALL)f (MPI_ISEND\(h1,)g(a,)g(1,)h(MPI_REAL,)e(1,)i(0,)g(comm,)f(ierr\))218 2172 y(CALL)g(MPI_ISEND\(h2,)g(b,)g(1,)h(MPI_REAL,)e(1,)i(0,)g(comm,)f (ierr\))147 2229 y(ELSE)94 b(!)24 b(rank.EQ.1)218 2285 y(CALL)f(MPI_IRECV\()g(h1,)g(a,)h(1,)g(MPI_REAL,)e(0,)i(MPI_ANY_TAG,)e (comm,)h(ierr\))218 2342 y(CALL)g(MPI_IRECV\()g(h2,)g(b,)h(1,)g (MPI_REAL,)e(0,)i(0,)f(comm,)h(ierr\))75 2398 y(END)f(IF)75 2454 y(CALL)g(MPI_WAIT\(h1,status\))75 2511 y(CALL)g (MPI_WAIT\(h2,status\))166 2591 y FC(The)17 b(\014rst)f(send)h(of)f (pro)q(cess)h(zero)g(will)h(matc)o(h)e(the)h(\014rst)f(receiv)o(e)i(of) e(pro)q(cess)h(one,)g(ev)o(en)g(if)g(b)q(oth)75 2647 y(messages)e(are)h(sen)o(t)f(b)q(efore)h(pro)q(cess)g(one)g(executes)h (either)f(receiv)o(e.)22 b(This)17 b(precludes)g(an)f(implemen-)75 2704 y(tation)f(where)g(di\013eren)o(t)h(queues)g(are)f(used)g(to)g(p)q (ost)g(the)g(t)o(w)o(o)f(receiv)o(es.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 40 32 39 bop 75 -100 a FC(32)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(A)22 b(call)g(to)f Fv(MPI)p 458 45 14 2 v 16 w(W)l(AIT)h FC(that)f (completes)h(a)f(receiv)o(e)i(will)g(ev)o(en)o(tually)g(terminate)e (and)h(return)75 102 y(if)f(a)f(matc)o(hing)g(send)h(has)f(b)q(een)h (started,)g(unless)g(the)g(send)g(is)f(satis\014ed)h(b)o(y)f(another)g (receiv)o(e.)36 b(In)75 158 y(particular,)20 b(if)f(the)g(matc)o(hing)g (send)g(is)g(non)o(blo)q(c)o(king,)i(then)e(the)g(receiv)o(e)g(should)h (complete)f(ev)o(en)g(if)75 214 y(no)c(call)i(is)e(executed)i(b)o(y)e (the)g(sender)h(to)f(complete)h(the)f(send.)20 b(Similarly)l(,)e(a)d (call)h(to)f Fv(MPI)p 1651 214 V 15 w(W)l(AIT)h FC(that)75 271 y(completes)i(a)e(send)i(will)h(ev)o(en)o(tually)f(return)f(if)g(a) g(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)h(started,)f(unless)h(the)75 327 y(receiv)o(e)e(is)g(satis\014ed)f(b)o(y)g(another)g(send,)g(and)g (ev)o(en)h(if)f(no)g(call)h(is)g(executed)g(to)e(complete)i(the)f (receiv)o(e.)166 386 y(Example:)75 507 y Fo(CALL)23 b (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 564 y(IF)h(\(RANK.EQ.0\))147 620 y(THEN)218 677 y(CALL)f(MPI_ISEND\()g(h,)h(a,)f(1,)h(MPI_REAL,)e (1,)i(0,)g(comm,)f(ierr\))218 733 y(CALL)g(MPI_RECV\()g(b,)h(1,)f (MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 790 y(CALL)g(MPI_WAIT\()g(h,) h(status\))147 846 y(ELSE)94 b(!)24 b(rank.EQ.1)218 903 y(CALL)f(MPI_RECV\()g(a,)h(1,)f(MPI_REAL,)g(0,)h(0,)f(comm,)g(ierr\)) 218 959 y(CALL)g(MPI_SEND\()g(b,)h(1,)f(MPI_REAL,)g(0,)h(0,)f(comm,)g (ierr\))75 1015 y(END)g(IF)166 1134 y FC(This)16 b(co)q(de)f(should)i (not)d(deadlo)q(c)o(k)j(in)f(a)f(correct)f(MPI)h(implemen)o(tation.)166 1193 y(If)j Fv(MPI)p 299 1193 V 16 w(TEST)g FC(that)e(completes)j(a)e (receiv)o(e)i(is)f(rep)q(eatedly)h(called)g(with)f(the)g(same)f (argumen)o(ts,)75 1249 y(and)d(a)g(matc)o(hing)g(send)g(has)g(b)q(een)h (started,)e(then)h(the)g(call)h(will)g(ev)o(en)o(tually)g(return)f Fv(\015ag=true)p FC(,)h(unless)75 1306 y(the)i(send)g(is)g(satis\014ed) h(b)o(y)f(another)f(receiv)o(e.)25 b(If)17 b Fv(MPI)p 1020 1306 V 16 w(TEST)g FC(that)f(completes)i(a)e(send)i(is)f(rep)q (eatedly)75 1362 y(called)e(with)f(the)g(same)f(argumen)o(ts,)g(and)h (a)f(matc)o(hing)g(receiv)o(e)i(has)e(b)q(een)i(started,)e(then)h(the)g (call)g(will)75 1419 y(ev)o(en)o(tually)i(return)f Fv(\015ag=true)p FC(,)h(unless)g(the)g(receiv)o(e)g(is)f(satis\014ed)h(b)o(y)f(another)g (send.)166 1478 y(Non)o(blo)q(c)o(king)g(comm)o(unications)g(cannot)f (b)q(e)g(replaced)i(b)o(y)e(blo)q(c)o(king)h(comm)o(unications,)g(ev)o (en)f(in)75 1534 y(the)h Fo(synchronous)f FC(comm)o(unication)i(mo)q (de.)k(Consider)c(the)f(follo)o(wing)h(example:)75 1640 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1697 y(IF\(rank.EQ.0\))123 1753 y(THEN)170 1810 y(CALL)h (MPI_ISEND\(request1,)d(a\(1\),)i(count,)g(MPI_REAL,)g(1,)g(tag1,)h (comm,)f(ierr\))170 1866 y(CALL)h(MPI_ISEND\(request2,)d(a\(2\),)i (count,)g(MPI_REAL,)g(1,)g(tag2,)h(comm,)f(ierr\))170 1922 y(CALL)h(MPI_WAIT\(request1,)d(status,)i(ierr\))170 1979 y(CALL)h(MPI_WAIT\(request2,)d(status,)i(ierr\))123 2035 y(ELSE)170 2092 y(CALL)h(MPI_IRECV\(request2,)d(a\(2\),)i(count,)g (MPI_REAL,)g(0,)g(tag2,)h(comm,)f(ierr\))170 2148 y(CALL)h (MPI_IRECV\(request1,)d(a\(1\),)i(count,)g(MPI_REAL,)g(0,)g(tag1,)h (comm,)f(ierr\))170 2205 y(CALL)h(MPI_WAIT\(request2,)d(status,)i (ierr\))170 2261 y(CALL)h(MPI_WAIT\(request1,)d(status,)i(ierr\))75 2318 y(END)g(IF)166 2421 y FC(The)16 b(co)q(de)g(is)g(guaran)o(teed)f (to)g(execute)h(correctly)l(,)g(ev)o(en)g(in)g(an)g(implemen)o(tation)h (that)d(do)q(es)i(not)75 2478 y(bu\013er)d(messages)f(\(the)h(only)h (resource)f(requiremen)o(t)g(is)h(the)f(abilit)o(y)h(to)e(ha)o(v)o(e)h (t)o(w)o(o)f(p)q(ending)i(comm)o(uni-)75 2534 y(cations\).)20 b(If)15 b(one)h(replaces)g(the)f(non)o(blo)q(c)o(king)i(sends)f (\(receiv)o(es\))f(b)o(y)g(blo)q(c)o(king)i(sends)f(\(receiv)o(es\),)f (and)75 2591 y(deletes)i(the)f(calls)g(to)f Fv(MPI)p 546 2591 V 16 w(W)l(AIT)p FC(,)h(then)g(the)f(co)q(de)i(will)g(deadlo)q (c)o(k.)22 b(The)16 b(co)q(de)h(will)g(also)f(deadlo)q(c)o(k)g(if)75 2647 y(one)i(deletes)i(the)e(calls)i(that)d(initiate)j(comm)o (unication)f(and)f(replaces)h(eac)o(h)g(call)g(to)f Fv(MPI)p 1683 2647 V 16 w(W)l(AIT)g FC(b)o(y)75 2704 y(the)d(appropriate)g(blo)q (c)o(king)i(send)f(or)f(receiv)o(e.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 41 33 40 bop 75 -100 a Fs(3.8.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b FC(33)75 45 y Fp(3.8.5)49 b(Multiple)17 b(Completions)75 134 y FC(It)23 b(is)h(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)g(to)e(w) o(ait)h(for)g(the)g(completion)i(of)d(an)o(y)h(or)g(all)h(the)g(op)q (erations)f(in)75 191 y(a)g(set,)h(rather)e(than)h(ha)o(ving)g(to)f(w)o (ait)h(for)f(a)g(sp)q(eci\014c)j(message.)43 b(A)23 b(call)h(to)e Fv(MPI)p 1597 191 14 2 v 16 w(W)l(AIT)l(ANY)h FC(or)75 247 y Fv(MPI)p 160 247 V 16 w(TEST)l(ANY)e FC(can)f(b)q(e)g(used)g(to)f (w)o(ait)h(for)f(the)h(completion)g(of)g(one)g(out)f(of)g(sev)o(eral)h (op)q(erations;)75 304 y(a)14 b(call)h(to)27 b Fv(MPI)p 347 304 V 15 w(W)l(AIT)l(ALL)14 b FC(or)f Fv(MPI)p 701 304 V 16 w(TEST)l(ALL)h FC(can)g(b)q(e)g(used)h(to)e(w)o(ait)g(for)h (all)g(p)q(ending)i(op)q(erations)e(in)75 360 y(a)h(list.)75 466 y Fv(MPI)p 160 466 V 16 w(W)l(AIT)l(ANY)h(\(count,)g(a)o(rra)o(y)p 640 466 V 14 w(of)p 691 466 V 16 w(requests,)h(index,)f(status\))117 545 y Fq(IN)155 b Fv(count)482 b Fq(list)14 b(length)f(\(in)o(teger\)) 117 624 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 624 V 15 w(of)p 468 624 V 16 w(requests)272 b Fq(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handle\))117 702 y(OUT)108 b Fv(index)489 b Fq(index)12 b(of)f(handle)g(for)g(op)q(eration)h(that)f(completed)g (\(in)o(teger\))117 781 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\))75 908 y Fo(int)23 b(MPI)p 245 908 15 2 v 17 w(Waitany\(int)g(count,)g(MPI)p 788 908 V 16 w(Comm)p 900 908 V 17 w(request)g(*array)p 1252 908 V 16 w(of)p 1316 908 V 17 w(requests,)g(int)g(*index,)393 964 y(MPI)p 468 964 V 17 w(Status)g(*status\))75 1052 y(MPI)p 150 1052 V 17 w(WAITANY\(COUNT,)f(ARRAY)p 645 1052 V 16 w(OF)p 709 1052 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g (IERROR\))170 1109 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1109 V 17 w(OF)p 716 1109 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 1109 V 16 w(STATUS)p 1610 1109 V 16 w(SIZE\),)170 1165 y(IERROR)166 1253 y FC(Blo)q(c)o(ks)17 b(un)o(til)h(one)e(of)g (the)h(op)q(erations)f(asso)q(ciated)h(with)g(the)f(comm)o(unication)h (requests)g(in)g(the)75 1310 y(arra)o(y)c(has)h(completed.)20 b(Returns)15 b(in)g Fv(index)g FC(the)f(index)h(of)f(that)f(request)h (in)h(the)f(arra)o(y)f(and)h(returns)g(in)75 1366 y Fv(status)j FC(the)e(status)g(of)g(the)g(completing)h(comm)o(unication.)166 1425 y(The)j(execution)h(of)e Fv(MPI)p 610 1425 14 2 v 16 w(W)l(AIT)l(ANY\(count,)j(a)o(rra)o(y)p 1079 1425 V 14 w(of)p 1130 1425 V 16 w(requests,)h(index,)e(status\))g FC(has)f(the)f(same)75 1481 y(e\013ect)10 b(as)g(the)g(execution)i(of) 20 b Fv(MPI)p 655 1481 V 16 w(W)l(AIT\(a)o(rra)o(y)p 897 1481 V 14 w(of)p 948 1481 V 16 w(requests[i],)13 b(status\))p FC(,)g(where)e Fv(i)f FC(is)h(the)f(v)m(alue)i(returned)75 1538 y(b)o(y)h Fv(index)p FC(.)21 b(In)14 b(particular,)g(the)g(asso)q (ciated)f(comm)o(unication)i(ob)s(ject)d(is)i(deallo)q(cated,)h(and)f (the)g(handle)75 1594 y(to)21 b(it)h(in)g Fv(a)o(rra)o(y)p 346 1594 V 15 w(of)p 398 1594 V 16 w(requests)i FC(is)e(set)f(to)g(n)o (ull.)41 b(If)22 b(more)f(then)h(one)g(op)q(eration)g(is)g(enabled)h (and)f(can)75 1651 y(terminate,)d(one)g(is)g(arbitrarily)g(c)o(hosen.) 30 b(There)19 b(is)g(no)g(requiremen)o(t)g(that)f(the)g(c)o(hoice)i (satisfy)e(an)o(y)75 1707 y(fairness)e(criterion.)75 1812 y Fv(MPI)p 160 1812 V 16 w(TEST)l(ANY\(count,)h(a)o(rra)o(y)p 626 1812 V 14 w(of)p 677 1812 V 16 w(requests,)g(index,)f(\015ag,)f (status\))117 1892 y Fq(IN)155 b Fv(count)482 b Fq(list)14 b(length)f(\(in)o(teger\))117 1970 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 1970 V 15 w(of)p 468 1970 V 16 w(requests)272 b Fq(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handle\))117 2049 y(OUT)108 b Fv(index)489 b Fq(index)9 b(of)g(op)q(eration)g(that)h (completed,)f(or)g Fl(MPI)p 1651 2049 13 2 v 14 w(UNDEFINED)905 2106 y Fq(if)k(none)h(completed)g(\(in)o(teger\))117 2184 y(OUT)108 b Fv(\015ag)518 b Fl(TRUE)13 b Fq(if)g(one)i(of)e(the)h (op)q(erations)h(is)e(complete)h(\(logical\))117 2263 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\))75 2389 y Fo(int)23 b(MPI)p 245 2389 15 2 v 17 w(Testany\(int)g(count,)g (MPI)p 788 2389 V 16 w(Comm)p 900 2389 V 17 w(request)g(*array)p 1252 2389 V 16 w(of)p 1316 2389 V 17 w(requests,)g(int)g(*index,)393 2446 y(int)h(*flag,)f(MPI)p 731 2446 V 16 w(Status)g(*status\))75 2534 y(MPI)p 150 2534 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 2534 V 16 w(OF)p 709 2534 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))170 2591 y(LOGICAL)g(FLAG)170 2647 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2647 V 17 w(OF)p 716 2647 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 2647 V 16 w(STATUS)p 1610 2647 V 16 w(SIZE\),)170 2704 y(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 42 34 41 bop 75 -100 a FC(34)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(Causes)11 b(either)h(one)f(or)g(none)h(of)f(the)g(op)q(erations)h(asso)q(ciated)f (with)h(the)f(comm)o(unication)h(requests)75 102 y(to)f(return.)18 b(In)12 b(the)f(former)g(case,)g(it)h(returns)f(in)h Fo(index)f FC(the)g(index)i(of)d(its)i(request)f(in)h(the)f(arra)o(y)l (,)g(returns)75 158 y(in)19 b Fo(status)e FC(the)h(status)f(of)h(that)f (op)q(eration,)i(deallo)q(cates)g(the)f(asso)q(ciated)g(comm)o (unication)h(ob)s(ject,)75 214 y(and)13 b(sets)f(the)h(request)f(to)g (n)o(ull.)21 b(In)13 b(the)f(latter)h(case,)f(it)h(returns)g(a)f(v)m (alue)i(of)e Fl(MPI)p 1469 214 13 2 v 14 w(UNDEFINED)g FC(in)h Fv(index)75 271 y FC(and)i Fv(status)j FC(is)d(unde\014ned.)166 329 y(The)k(execution)g(of)f Fv(MPI)p 609 329 14 2 v 16 w(TEST)l(ANY\(count,)j(a)o(rra)o(y)p 1079 329 V 15 w(of)p 1131 329 V 16 w(requests,)g(index,)f(status\))g FC(has)f(the)f(same)75 385 y(e\013ect)k(as)g(the)g(execution)h(of)f Fv(MPI)p 704 385 V 16 w(TEST\(a)o(rra)o(y)p 947 385 V 14 w(of)p 998 385 V 16 w(requests[i],)k(\015ag,)e(status\))p FC(,)h(for)d Fo(i=0,)h(1)h(,...,)75 442 y(count-1)p FC(,)e(in)g(some)f (arbitrary)g(order,)i(un)o(til)f(one)g(call)h(returns)e Fo(flag=true)p FC(,)h(or)f(all)h(fail.)40 b(In)22 b(the)75 498 y(former)12 b(case,)h Fo(index)f FC(is)h(set)g(to)f(the)h(last)f(v) m(alue)i(of)f Fo(i)p FC(;)g(in)g(the)g(later)g(case,)g(it)g(is)g(set)g (to)f Fl(MPI)p 1618 498 13 2 v 14 w(UNDEFINED)p FC(.)75 603 y Fv(MPI)p 160 603 14 2 v 16 w(W)l(AIT)l(ALL\()i(count,)i(a)o(rra)o (y)p 626 603 V 15 w(of)p 678 603 V 16 w(requests,)h(a)o(rra)o(y)p 971 603 V 15 w(of)p 1023 603 V 15 w(statuses\))117 681 y Fq(IN)155 b Fv(count)482 b Fq(lists)14 b(length)g(\(in)o(teger\))117 759 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 759 V 15 w(of)p 468 759 V 16 w(requests)272 b Fq(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handle\))117 837 y(OUT)108 b Fv(a)o(rra)o(y)p 416 837 V 15 w(of)p 468 837 V 16 w(statuses)276 b Fq(arra)o(y)14 b(of)f(status)i(ob)r(jects)g(\(Status\))75 962 y Fo(int)23 b(MPI)p 245 962 15 2 v 17 w(Waitall\(int)g(count,)g(MPI)p 788 962 V 16 w(Comm)p 900 962 V 17 w(request)g(*array)p 1252 962 V 16 w(of)p 1316 962 V 17 w(requests,)393 1019 y(MPI)p 468 1019 V 17 w(Status)g(*array)p 796 1019 V 16 w(of)p 860 1019 V 17 w(statuses\))75 1107 y(MPI)p 150 1107 V 17 w(WAITALL\(COUNT,)f(ARRAY)p 645 1107 V 16 w(OF)p 709 1107 V 17 w(REQUESTS,)h(ARRAY)p 1085 1107 V 16 w(OF)p 1149 1107 V 17 w(STATUSES,)g(IERROR\))170 1163 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1163 V 17 w(OF)p 716 1163 V 17 w(REQUESTS\(*\),)170 1220 y(ARRAY)p 293 1220 V 17 w(OF)p 358 1220 V 17 w(STATUSES\(MPI)p 663 1220 V 15 w(STATUS)p 822 1220 V 17 w(SIZE,*\),)g(IERROR)166 1307 y FC(Blo)q(c)o(ks)12 b(un)o(til)h(all)g(comm)o(unication)g(op)q (erations)f(asso)q(ciated)g(with)g(requests)f(in)i(the)f(list)h (complete,)75 1364 y(and)k(return)g(the)g(status)f(of)h(all)h(these)f (op)q(erations.)26 b(Both)16 b(arra)o(ys)g(ha)o(v)o(e)h(the)g(same)f(n) o(um)o(b)q(er)i(of)e(v)m(alid)75 1420 y(en)o(tries.)24 b(The)16 b Fv(i)p FC(-th)h(en)o(try)f(in)h Fv(a)o(rra)o(y)p 694 1420 14 2 v 14 w(of)p 745 1420 V 16 w(statuses)i FC(is)e(set)f(to)g(the)h(return)f(status)f(of)h(the)h Fv(i)p FC(-th)f(op)q(eration.)75 1477 y(All)g(comm)o(unication)g(ob)s (jects)f(are)g(deallo)q(cated)h(and)g(all)g(requests)f(are)g(set)g(to)g (n)o(ull.)166 1534 y(The)f(execution)h(of)f Fv(MPI)p 596 1534 V 16 w(W)l(AIT)l(ALL\(count,)g(a)o(rra)o(y)p 1046 1534 V 15 w(of)p 1098 1534 V 16 w(requests,)i(a)o(rra)o(y)p 1390 1534 V 15 w(of)p 1442 1534 V 16 w(statuses)g FC(has)e(the)g(same) 75 1591 y(e\013ect)j(as)g(the)g(execution)i(of)e Fv(MPI)p 680 1591 V 16 w(W)l(AIT\(a)o(rra)o(y)p 922 1591 V 14 w(of)p 973 1591 V 16 w(request[i],)i(a)o(rra)o(y)p 1288 1591 V 15 w(of)p 1340 1591 V 16 w(statuses[i]\))p FC(,)h(for)c Fo(i=0)24 b(,...,)75 1647 y(count-1)p FC(,)14 b(in)i(some)f(arbitrary)f (order.)75 1752 y Fv(MPI)p 160 1752 V 16 w(TEST)l(ALL\(count,)i(a)o (rra)o(y)p 613 1752 V 14 w(of)p 664 1752 V 16 w(requests,)h(\015ag,)e (a)o(rra)o(y)p 1054 1752 V 14 w(of)p 1105 1752 V 16 w(statuses\))117 1831 y Fq(IN)155 b Fv(count)482 b Fq(lists)14 b(length)g(\(in)o (teger\))117 1908 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 1908 V 15 w(of)p 468 1908 V 16 w(requests)272 b Fq(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handle\))117 1986 y(OUT)108 b Fv(\015ag)518 b Fq(\(logical\))117 2064 y(OUT)108 b Fv(a)o(rra)o(y)p 416 2064 V 15 w(of)p 468 2064 V 16 w(statuses)276 b Fq(arra)o(y)14 b(of)f(status)i(ob)r(jects)g (\(Status\))75 2189 y Fo(int)23 b(MPI)p 245 2189 15 2 v 17 w(Testall\(int)g(count,)g(MPI)p 788 2189 V 16 w(Comm)p 900 2189 V 17 w(request)g(*array)p 1252 2189 V 16 w(of)p 1316 2189 V 17 w(requests,)g(int)g(*flag,)393 2246 y(MPI)p 468 2246 V 17 w(Status)g(*array)p 796 2246 V 16 w(of)p 860 2246 V 17 w(statuses\))75 2334 y(MPI)p 150 2334 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 2334 V 16 w(OF)p 709 2334 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 2334 V 16 w(OF)p 1292 2334 V 17 w(STATUSES,)g(IERROR\))170 2390 y(LOGICAL)g(FLAG)170 2447 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2447 V 17 w(OF)p 716 2447 V 17 w(REQUESTS\(*\),)170 2503 y(ARRAY)p 293 2503 V 17 w(OF)p 358 2503 V 17 w(STATUSES\(MPI)p 663 2503 V 15 w(STATUS)p 822 2503 V 17 w(SIZE,*\),)g(IERROR)166 2591 y FC(Causes)13 b(either)g(all)h(or)f(none)g(of)g(the)g(op)q (erations)g(asso)q(ciated)g(with)g(the)g(comm)o(unication)h(requests)75 2647 y(to)k(complete.)30 b(It)18 b(returns)h Fv(\015ag)f(=)h(true)g FC(if)g(all)g(comm)o(unications)g(asso)q(ciated)g(with)f(requests)h(in) g(the)75 2704 y(arra)o(y)10 b(ha)o(v)o(e)h(completed.)19 b(In)12 b(this)g(case,)f(eac)o(h)h(status)e(en)o(try)h(is)h(set)f(to)f (the)i(status)e(of)h(the)g(corresp)q(onding)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 43 35 42 bop 75 -100 a Fs(3.8.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b FC(35)75 45 y(comm)o(unication.)20 b(All)14 b(comm)o(unication)g(ob)s (jects)e(are)g(deallo)q(cated,)j(and)e(all)h(requests)e(are)h(set)g(to) f(n)o(ull.)75 102 y(Otherwise,)h Fv(\015ag)f(=)g(false)f FC(is)i(returned,)f(no)g(request)g(is)g(mo)q(di\014ed)h(and)f(the)g(v)m (alues)h(of)e(the)h(status)f(en)o(tries)75 158 y(are)k(unde\014ned.)22 b(This)15 b(is)h(a)f(non)o(blo)q(c)o(king)i(op)q(eration)e(that)f (returns)h(immediately)l(.)166 214 y(Example:)75 363 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 419 y(IF\(rank)g(<)h(2\))123 476 y(THEN)214 b(!)24 b(rank=0)f(or)g (rank=1)170 532 y(CALL)h(MPI_ISEND\(request,)d(a,)j(n,)f(MPI_REAL,)g (2,)h(tag,)f(comm,)g(ierr\))170 589 y(****)h(do)f(some)h(computation)e (to)i(mask)f(latency)g(****)170 645 y(CALL)h(MPI_WAIT\(request,)d (status,)i(ierr\))123 702 y(ELSE)214 b(!)24 b(rank=2)170 758 y(CALL)g(MPI_IRECV\(request_list\(0)o(\),)d(a,)j(n,)f(MPI_REAL,)g (0,)g(tag,)h(comm,)f(ierr\))170 814 y(CALL)h (MPI_IRECV\(request_list\(1)o(\),)d(b,)j(n,)f(MPI_REAL,)g(1,)g(tag,)h (comm,)f(ierr\))170 871 y(****)h(do)f(some)h(computation)e(to)i(mask)f (latency)g(****)170 927 y(CALL)h(MPI_WAITANY\(2,)e(request_list,)g (index,)h(status,)g(ierr\))170 984 y(IF\(index.EQ.0\))218 1040 y(THEN)266 1097 y(****)g(handle)g(message)g(from)g(process)g(0)h (****)266 1153 y(CALL)f(MPI_WAIT\(request_list\(1\),)e(status,)i (ierr\))266 1210 y(****)g(handle)g(message)g(from)g(process)g(1)h(****) 218 1266 y(ELSE)266 1323 y(****)f(handle)g(message)g(from)g(process)g (1)h(****)266 1379 y(CALL)f(MPI_WAIT\(request_list\(0\),)e(status,)i (ierr\))266 1435 y(****)g(handle)g(message)g(from)g(process)g(0)h(****) 170 1492 y(END)g(IF)75 1548 y(END)f(IF)166 1640 y FC(The)13 b(calls)h(in)o(tro)q(duced)f(in)h(this)f(subsection)h(can)f(b)q(e)g (used)g(to)f(w)o(ait)g(or)h(test)f(for)g(the)h(completion)g(of)75 1697 y(an)h(arbitrary)f(mix)h(of)f(non)o(blo)q(c)o(king)i(comm)o (unication)f(calls,)h(including)h(a)e(mix)g(of)f(sends)h(and)g(receiv)o (es)75 1753 y(and)h(of)g(an)o(y)g(additional)i(non)o(blo)q(c)o(king)f (comm)o(unication)g(calls)g(pro)o(vided)g(b)o(y)f(MPI.)166 1810 y(The)j Fv(status)i FC(argumen)o(t)d(in)i Fv(MPI)p 740 1810 14 2 v 16 w(W)l(AIT)l(ANY)f FC(and)h Fv(MPI)p 1149 1810 V 15 w(W)l(AIT)l(ALL)f FC(and)g(the)g(en)o(tries)g(of)g(the)g Fv(a)o(r-)75 1866 y(ra)o(y)p 136 1866 V 15 w(of)p 188 1866 V 16 w(statuses)d FC(argumen)o(t)c(in)h Fv(MPI)p 698 1866 V 16 w(W)l(AIT)l(ALL)g FC(and)g Fv(MPI)p 1082 1866 V 16 w(TEST)l(ALL)f FC(are)h(not)f(used)i(or)e(up)q(dated)i(if)f (the)75 1923 y(completing)k(comm)o(unication)f(is)g(a)f(send.)21 b(If)14 b(all)i(op)q(erations)f(in)g(the)g(list)g(pro)o(vided)h(b)o(y)e (the)h(argumen)o(t)75 1979 y Fv(a)o(rra)o(y)p 173 1979 V 15 w(of)p 225 1979 V 16 w(requests)20 b FC(are)e(send)h(op)q (erations,)f(then)h(an)f(arbitrary)g(v)m(alue)h(can)g(b)q(e)g(passed)f (to)g(the)g Fv(status)75 2035 y FC(\(resp.)35 b Fv(a)o(rra)o(y)p 320 2035 V 14 w(of)p 371 2035 V 16 w(statuses)p FC(\))18 b(argumen)o(t.)166 2092 y(An)j Fv(a)o(rra)o(y)p 344 2092 V 15 w(of)p 396 2092 V 16 w(requests)i FC(argumen)o(t)e(that)f(is)i (passed)g(to)e(an)o(y)h(of)g(the)h(last)f(four)g(functions)h(ma)o(y)75 2148 y(con)o(tain)14 b(n)o(ull)i(en)o(tries.)k(These)14 b(en)o(tries)h(are)e(ignored)i(and)f(the)h(call)g(executes)g(as)e(if)i (it)f(w)o(as)g(supplied)i(an)75 2205 y(arra)o(y)g(of)h(requests)h(con)o (taining)g(only)g(the)g(non-n)o(ull)h(en)o(tries.)27 b(As)17 b(a)h(request)f(in)h Fv(a)o(rra)o(y)p 1608 2205 V 15 w(of)p 1660 2205 V 16 w(requests)h FC(is)75 2261 y(set)13 b(to)g(n)o(ull)i(when)f(the)g(corresp)q(onding)g(op)q(eration) g(completes,)g(this)g(allo)o(ws)g(to)f(reuse)g(the)h(same)f(arra)o(y)75 2318 y(for)j(completing)h(in)g(successiv)o(e)g(calls)h(the)e(op)q (erations)g(asso)q(ciated)h(with)f(this)h(arra)o(y)l(.)22 b(Th)o(us,)16 b(the)g(last)75 2374 y(example)g(can)f(b)q(e)h(rewritten) f(as)g(follo)o(ws:)75 2534 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,) h(ierr\))75 2591 y(IF\(rank)g(<)h(2\))123 2647 y(THEN)214 b(!)24 b(rank=0)f(or)g(rank=1)170 2704 y(CALL)h(MPI_ISEND\(request,)d (a,)j(n,)f(MPI_REAL,)g(2,)h(tag,)f(comm,)g(ierr\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 44 36 43 bop 75 -100 a FC(36)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)170 45 y Fo(CALL)24 b(MPI_WAIT\(request,)d(status,)i(ierr\))123 102 y(ELSE)214 b(!)24 b(rank=2)170 158 y(CALL)g(MPI_IRECV\(request_list\(0)o(\),)d(a,) j(n,)f(MPI_REAL,)g(0,)g(tag,)h(comm,)f(ierr\))170 214 y(CALL)h(MPI_IRECV\(request_list\(1)o(\),)d(b,)j(n,)f(MPI_REAL,)g(1,)g (tag,)h(comm,)f(ierr\))170 271 y(CALL)h(MPI_WAITANY\(2,)e (request_list,)g(index,)h(status,)g(ierr\))266 327 y(****)g(handle)g (first)g(arriving)g(message)47 b(****)170 384 y(CALL)24 b(MPI_WAITANY\(2,)e(request_list,)g(index,)h(status,)g(ierr\))266 440 y(****)g(handle)g(second)g(arriving)g(message)47 b(****)166 564 y FC(If)18 b Fv(MPI)p 299 564 14 2 v 16 w(W)l(AIT)l(ANY)i FC(or)d Fv(MPI)p 676 564 V 16 w(TEST)l(ANY)j FC(are)e(called)i(with)e(an)g(empt)o(y)g Fv(a)o(rra)o(y)p 1534 564 V 15 w(of)p 1586 564 V 16 w(requests)i FC(argu-)75 620 y(men)o(t)d(\(i.e.,)g(an)g(arra)o(y)f(of)h(length)h(zero,)f(or)f (an)i(arra)o(y)d(with)j(all)g(en)o(tries)g(n)o(ull\))g(then)f(they)h (return)f(the)75 677 y(v)m(alue)k Fl(MPI)p 274 677 13 2 v 14 w(UNDEFINED)d FC(in)i Fv(index)p FC(;)i Fv(MPI)p 812 677 14 2 v 16 w(W)l(AIT)l(ALL)d FC(with)h(an)f(empt)o(y)g Fv(a)o(rra)o(y)p 1449 677 V 14 w(of)p 1500 677 V 16 w(requests)j FC(argumen)o(t)75 733 y(has)15 b(no)g(e\013ect;)g Fv(MPI)p 437 733 V 16 w(TEST)l(ALL)g FC(with)g(an)g(empt)o(y)g Fv(a)o(rra)o(y)p 1058 733 V 15 w(of)p 1110 733 V 16 w(requests)i FC(argumen)o(t)e(returns)g Fv(\015ag=true)p FC(.)75 896 y Ft(3.9)59 b(Prob)r(e)20 b(and)f(Cancel)75 1004 y FC(The)d Fv(MPI)p 254 1004 V 16 w(PROBE)g FC(and)g Fv(MPI)p 605 1004 V 16 w(IPROBE)g FC(op)q(erations)g(allo)o(w)g(incoming)h(messages) e(to)g(b)q(e)h(c)o(hec)o(k)o(ed)g(for,)75 1060 y(without)g(actually)h (receiving)h(them.)k(The)17 b(user)f(can)g(then)h(decide)g(ho)o(w)f(to) g(receiv)o(e)h(them,)f(based)g(on)75 1117 y(the)i(information)f (returned)h(b)o(y)f(the)h(prob)q(e)g(\(basically)l(,)h(the)e (information)h(returned)g(b)o(y)f Fv(status)p FC(\).)28 b(In)75 1173 y(particular,)16 b(the)g(user)g(ma)o(y)f(allo)q(cate)h (memory)f(for)g(the)h(receiv)o(e)h(bu\013er,)e(according)h(to)f(the)h (length)g(of)75 1230 y(the)f(prob)q(ed)h(message.)166 1289 y(An)24 b(MPI)f(implemen)o(tation)h(of)f Fv(MPI)p 838 1289 V 16 w(PROBE)i FC(needs)f(to)e(guaran)o(tee)h(progress:)35 b(if)24 b(a)f(call)i(to)75 1346 y Fv(MPI)p 160 1346 V 16 w(PROBE)20 b FC(has)f(b)q(een)h(issued)h(b)o(y)e(a)f(pro)q(cess,)i (and)g(a)e(send)i(that)e(matc)o(hes)h(the)g(prob)q(e)h(has)f(b)q(een)75 1402 y(initiated)g(b)o(y)e(some)g(pro)q(cess,)h(then)g(the)f(call)i(to) e Fv(MPI)p 1029 1402 V 15 w(PROBE)i FC(will)g(return,)f(unless)g(the)g (message)e(is)75 1459 y(receiv)o(ed)21 b(b)o(y)e(another)g(concurren)o (t)h(receiv)o(e)h(op)q(eration)e(\(that)g(is)h(executed)g(b)o(y)g (another)f(thread)h(at)75 1515 y(the)c(probing)g(pro)q(cess\).)21 b(Similarly)l(,)d(if)e(a)g(pro)q(cess)g(busy)g(w)o(aits)f(with)h Fv(MPI)p 1374 1515 V 16 w(IPROBE)h FC(and)f(a)f(matc)o(hing)75 1572 y(message)f(has)g(b)q(een)h(issued,)h(then)e(the)h(call)g(to)f Fv(MPI)p 984 1572 V 16 w(IPROBE)h FC(will)h(ev)o(en)o(tually)f(return)g Fo(flag)23 b(=)h(true)75 1628 y FC(unless)16 b(the)g(message)e(is)i (receiv)o(ed)g(b)o(y)g(another)e(concurren)o(t)i(receiv)o(e)g(op)q (eration.)166 1688 y(The)h Fv(MPI)p 346 1688 V 16 w(CANCEL)g FC(op)q(eration)g(allo)o(ws)g(p)q(ending)i(comm)o(unications)e(to)f(b)q (e)i(cancelled.)27 b(This)17 b(is)75 1744 y(required)j(for)f(clean)o (up.)33 b(P)o(osting)19 b(a)g(send)h(or)f(a)g(receiv)o(e)h(ties)f(up)h (user)f(resources)h(\(send)f(or)g(receiv)o(e)75 1801 y(bu\013ers\),)14 b(and)i(a)f(cancel)h(ma)o(y)f(b)q(e)g(needed)i(to)d (free)i(these)f(resources)g(gracefully)l(.)75 1908 y Fv(MPI)p 160 1908 V 16 w(IPROBE\(source,)h(tag,)f(comm,)d(\015ag,)j (status\))117 1989 y Fq(IN)155 b Fv(source)468 b Fq(source)15 b(rank,)f(or)27 b Fl(MPI)p 1281 1989 13 2 v 15 w(ANY)p 1381 1989 V 14 w(SOURCE)13 b Fq(\(in)o(teger\))117 2071 y(IN)155 b Fv(tag)526 b Fq(tag)14 b(v)n(alue)f(or)28 b Fl(MPI)p 1226 2071 V 14 w(ANY)p 1325 2071 V 15 w(T)m(A)o(G)12 b Fq(\(in)o(teger\))117 2153 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 2235 y(OUT)108 b Fv(\015ag)518 b Fq(\(logical\))117 2316 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\))75 2444 y Fo(int)23 b(MPI)p 245 2444 15 2 v 17 w(Iprobe\(int)g(source,)g (int)g(tag,)g(MPI)p 1002 2444 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 2501 y(MPI)p 468 2501 V 17 w(Status)g(*status\))75 2591 y(MPI)p 150 2591 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 2647 y(LOGICAL)g(FLAG)170 2704 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 2704 V 16 w(STATUS)p 1218 2704 V 16 w(SIZE\),)g(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 45 37 44 bop 75 -100 a Fs(3.9.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b FC(37)166 45 y Fv(MPI)p 251 45 14 2 v 16 w(IPROBE)19 b FC(returns)f Fv(\015ag)g(=)h(true)f FC(if)h(there)g(is)f(a)g(message) g(that)g(can)g(b)q(e)h(receiv)o(ed)g(and)g(that)75 102 y(matc)o(hes)i(the)g(pattern)f(sp)q(eci\014ed)j(b)o(y)e(the)h(argumen)o (ts)e Fv(source)p FC(,)i Fv(tag)p FC(,)g(and)g Fv(comm)p FC(.)35 b(It)21 b(returns)g Fv(\015ag)75 158 y(=)i(false)p FC(,)h(otherwise.)41 b(If)23 b Fv(MPI)p 633 158 V 16 w(IPROBE)g FC(returns)f Fv(\015ag)g(=)h(true)p FC(,)h(then)f(the)f (status)g(ob)s(ject)g(can)g(b)q(e)75 214 y(subsequen)o(tly)c(queried)g (using)g(the)f(functions)h(describ)q(ed)g(in)g(section)g(3.3.1)d(to)h (\014nd)i(the)f(source,)g(tag)75 271 y(and)f(length)h(of)f(the)g (message.)23 b(The)16 b(v)m(alues)i(returned)e(b)o(y)h(these)f(queries) h(are)f(the)g(same)g(v)m(alues)i(that)75 327 y(w)o(ould)12 b(ha)o(v)o(e)e(b)q(een)j(returned)e(b)o(y)h(a)f(call)h(to)e Fv(MPI)p 897 327 V 16 w(RECV)i FC(executed)g(at)f(the)g(same)g(p)q(oin) o(t)h(in)g(the)f(program.)75 384 y(The)k(return)h(status)e(ob)s(ject)h (is)g(unde\014ned)i(if)f Fv(\015ag)f(=)h(false)p FC(.)166 441 y(A)d(subsequen)o(t)i(receiv)o(e)f(executed)g(with)g(the)g(same)f (con)o(text,)g(and)g(the)h(source)f(and)h(tag)f(returned)75 497 y(b)o(y)g(querying)h(the)f(status)f(returned)i(b)o(y)f Fv(MPI)p 858 497 V 16 w(IPROBE)h FC(will)h(receiv)o(e)f(the)f(message)f (that)h(w)o(as)f(matc)o(hed)75 554 y(b)o(y)21 b(the)g(prob)q(e,)h(if)g (no)f(other)f(in)o(terv)o(ening)i(receiv)o(e)g(o)q(ccurred)f(after)g (the)g(prob)q(e.)37 b(If)21 b(the)g(receiving)75 610 y(pro)q(cess)e(is)h(m)o(ulti-threaded,)h(it)f(is)f(the)h(user's)f(resp) q(onsibilit)o(y)i(to)e(ensure)h(that)e(the)i(last)f(condition)75 667 y(holds.)75 771 y Fv(MPI)p 160 771 V 16 w(PROBE\(source,)d(tag,)f (comm,)e(status\))117 848 y Fq(IN)155 b Fv(source)468 b Fq(source)15 b(rank,)f(or)27 b Fl(MPI)p 1281 848 13 2 v 15 w(ANY)p 1381 848 V 14 w(SOURCE)13 b Fq(\(in)o(teger\))117 924 y(IN)155 b Fv(tag)526 b Fq(tag)14 b(v)n(alue,)e(or)28 b Fl(MPI)p 1237 924 V 14 w(ANY)p 1336 924 V 15 w(T)m(A)o(G)13 b Fq(\(in)o(teger\))117 1000 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 1077 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\))75 1201 y Fo(int)23 b(MPI)p 245 1201 15 2 v 17 w(Probe\(int)g(source,)g (int)g(tag,)h(MPI)p 979 1201 V 16 w(Comm)g(comm,)f(MPI)p 1330 1201 V 17 w(Status)g(*status\))75 1288 y(MPI)p 150 1288 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g(IERROR\))170 1345 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1345 V 16 w(STATUS)p 1218 1345 V 16 w(SIZE\),)g(IERROR)166 1432 y Fv(MPI)p 251 1432 14 2 v 16 w(PROBE)17 b FC(b)q(eha)o(v)o(es)g (lik)o(e)g Fv(MPI)p 769 1432 V 16 w(IPROBE)g FC(except)g(that)f(it)g (is)h(a)f(blo)q(c)o(king)i(call)f(whic)o(h)g(returns)75 1488 y(only)f(after)e(a)h(matc)o(hing)g(message)g(has)g(b)q(een)h (found.)75 1592 y Fv(MPI)p 160 1592 V 16 w(CANCEL\(request\))117 1670 y Fq(IN)155 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 1795 y Fo(int)23 b(MPI)p 245 1795 15 2 v 17 w(Cancel\(MPI)p 502 1795 V 16 w(Comm)p 614 1795 V 17 w(request)g(request\))75 1882 y(MPI)p 150 1882 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1938 y(INTEGER)h(REQUEST,)g (IERROR)166 2025 y FC(A)17 b(call)h(to)e Fv(MPI)p 444 2025 14 2 v 15 w(CANCEL)i FC(marks)e(for)g(cancellation)i(a)f(p)q (ending)h(non)o(blo)q(c)o(king)g(comm)o(unication)75 2082 y(op)q(eration)f(\(send)g(or)g(receiv)o(e\).)25 b(The)18 b(cancel)g(call)g(is)f(non)o(blo)q(c)o(king,)h(and)f(lo)q (cal.)27 b(It)17 b(returns)f(immedi-)75 2138 y(ately)l(,)f(p)q(ossibly) i(b)q(efore)e(the)h(comm)o(unication)g(is)f(actually)h(cancelled.)166 2195 y(It)c(is)g(still)h(necessary)f(to)g(complete)g(a)g(comm)o (unication)g(that)f(has)h(b)q(een)h(mark)o(ed)e(for)h(cancellation,)75 2251 y(using)i(a)g(call)h(to)e Fv(MPI)p 450 2251 V 15 w(W)l(AIT)h FC(or)f Fv(MPI)p 728 2251 V 16 w(TEST)h FC(\(or)f(an)o(y)g (of)g(the)h(deriv)o(ed)h(op)q(erations\).)k(If)14 b(the)g(op)q(eration) 75 2308 y(has)h(b)q(een)h(cancelled,)g(then)f(information)g(to)g(that)f (e\013ect)g(will)j(b)q(e)e(returned)g(in)h(the)f(status)f(argumen)o(t) 75 2364 y(of)g(the)g(op)q(eration)g(that)f(completes)i(the)f(comm)o (unication.)20 b(If)14 b(a)g(comm)o(unication)g(is)h(mark)o(ed)e(for)h (can-)75 2421 y(cellation,)f(then)f(a)f Fv(MPI)p 485 2421 V 16 w(W)l(AIT)h FC(call)g(for)f(that)g(comm)o(unication)h(is)g (guaran)o(teed)f(to)g(return,)g(irresp)q(ectiv)o(e)75 2477 y(of)17 b(the)g(activities)i(of)e(other)g(pro)q(cesses)h(\(i.e.,)f Fv(MPI)p 970 2477 V 16 w(W)l(AIT)g FC(b)q(eha)o(v)o(es)h(as)f(a)g(lo)q (cal)i(function\);)f(similarly)75 2534 y(if)f Fv(MPI)p 203 2534 V 16 w(TEST)f FC(is)h(rep)q(eatedly)h(called)g(in)f(a)f(busy)g (w)o(ait)g(lo)q(op)h(for)f(a)g(cancelled)j(comm)o(unication,)e(then)75 2590 y Fv(MPI)p 160 2590 V 16 w(TEST)e FC(will)i(ev)o(en)o(tually)f(b)q (e)g(successful.)166 2647 y(Either)j(the)f(cancellation)i(succeeds,)f (or)f(the)g(comm)o(unication)h(succeeds,)h(but)e(not)g(b)q(oth.)29 b(If)18 b(a)75 2704 y(send)h(is)h(mark)o(ed)e(for)g(cancellation,)j (then)e(it)g(m)o(ust)f(b)q(e)i(the)f(case)g(that)f(either)h(the)g(send) g(completes)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 46 38 45 bop 75 -100 a FC(38)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FC(normally)l(,)i (in)h(whic)o(h)f(case)f(the)h(message)f(sen)o(t)g(w)o(as)g(receiv)o(ed) h(at)f(the)h(destination)g(pro)q(cess,)g(or)f(that)75 102 y(the)e(send)h(is)f(successfully)i(cancelled,)g(in)e(whic)o(h)h (case)f(no)g(part)f(of)h(the)g(message)f(w)o(as)g(receiv)o(ed)j(at)d (the)75 158 y(destination.)21 b(Then,)15 b(an)o(y)f(matc)o(hing)h (receiv)o(e)h(has)e(to)h(b)q(e)g(satis\014ed)h(b)o(y)e(another)h(send.) 20 b(If)15 b(a)g(receiv)o(e)g(is)75 214 y(mark)o(ed)d(for)f (cancellation,)k(then)d(it)h(m)o(ust)e(b)q(e)i(the)g(case)f(that)f (either)i(the)g(receiv)o(e)g(completes)g(normally)l(,)75 271 y(or)k(that)g(the)h(receiv)o(e)h(is)f(successfully)i(cancelled,)g (in)f(whic)o(h)f(case)g(no)g(part)f(of)g(the)h(receiv)o(e)h(bu\013er)e (is)75 327 y(altered.)j(Then,)c(an)o(y)e(matc)o(hing)i(send)g(has)f(to) f(b)q(e)i(satis\014ed)g(b)o(y)f(another)g(receiv)o(e.)75 431 y Fv(MPI)p 160 431 14 2 v 16 w(TEST)p 290 431 V 16 w(CANCELLED\(status,)i(\015ag\))117 508 y Fq(IN)155 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\))117 584 y(IN)155 b Fv(\015ag)518 b Fq(\(logical\))75 708 y Fo(int)23 b(MPI)p 245 708 15 2 v 17 w(Test)p 358 708 V 17 w(cancelled\(MPI)p 687 708 V 15 w(Status)g(status,)g(int)h (*flag\))75 795 y(MPI)p 150 795 V 17 w(TEST)p 263 795 V 16 w(CANCELLED\(STATUS,)e(FLAG,)h(IERROR\))170 851 y(LOGICAL)g(FLAG)170 908 y(INTEGER)g(STATUS\(MPI)p 604 908 V 16 w(STATUS)p 764 908 V 17 w(SIZE\),)g(IERROR)166 994 y FC(Returns)11 b Fv(\015ag)f(=)h(true)g FC(if)g(the)f(comm)o (unication)h(asso)q(ciated)g(with)f(the)h(status)e(ob)s(ject)h(w)o(as)g (cancelled)75 1051 y(successfully)l(.)27 b(In)18 b(suc)o(h)f(a)g(case,) g(all)g(other)g(\014elds)h(of)e Fv(status)j FC(\(suc)o(h)e(as)g Fv(count)h FC(or)e Fv(tag)p FC(\))h(are)f(unde\014ned.)75 1107 y(Returns)h Fv(\015ag)g(=)f(false)p FC(,)h(otherwise.)24 b(If)17 b(a)g(receiv)o(e)g(op)q(eration)g(migh)o(t)f(b)q(e)h(cancelled) i(then)e(one)g(should)75 1164 y(call)39 b Fv(MPI)p 266 1164 14 2 v 16 w(TEST)p 396 1164 V 17 w(CANCELLED)19 b FC(\014rst,)g(to)f(c)o(hec)o(k)i(whether)f(the)g(op)q(eration)g(w)o (as)g(cancelled,)j(b)q(efore)75 1220 y(c)o(hec)o(king)16 b(on)f(the)g(other)g(\014elds)i(of)d(the)i(return)f(status.)166 1359 y Fn(Implemen)o(tati)o(on)h(note:)51 b Fq(If)18 b(a)f(send)h(op)q(eration)g(uses)h(an)e(\\eager")h(proto)q(col)f (\(data)h(is)f(transferred)75 1416 y(to)e(the)h(receiv)o(er)h(b)q (efore)f(a)g(matc)o(hing)d(receiv)o(e)k(is)e(p)q(osted\),)i(then)f(the) g(cancelation)f(of)g(this)g(send)i(ma)o(y)c(require)75 1472 y(comm)o(unication)j(with)j(the)h(in)o(tended)g(receiv)o(er,)h(in) e(order)h(to)f(free)h(allo)q(cated)f(bu\013ers.)36 b(On)19 b(some)g(systems)75 1529 y(this)14 b(ma)o(y)e(require)j(an)f(in)o (terrupt)h(to)e(the)i(in)o(tended)g(receiv)o(er.)20 b(Therefore,)15 b(it)e(is)h(exp)q(ected)i(that)e Fl(MPI)p 1705 1529 13 2 v 15 w(CANCEL)75 1585 y Fq(will)h(b)q(e)i(an)f(exp)q(ensiv)o(e)i(op)q (eration)e(on)g(man)o(y)f(systems,)i(to)f(b)q(e)h(used)g(only)f (exceptionally)m(.)25 b(Note)17 b(that,)f(while)75 1642 y(comm)o(unication)f(ma)o(y)h(b)q(e)i(needed)i(to)e(implemen)o(t)d Fl(MPI)p 1006 1642 V 14 w(CANCEL)p Fq(,)i(this)h(is)g(still)f(a)h(lo)q (cal)f(op)q(eration,)i(in)e(the)75 1698 y(sense)h(that)e(its)g (completion)e(do)q(es)j(not)f(dep)q(end)h(on)f(the)g(co)q(de)h (executed)h(b)o(y)e(other)g(pro)q(cesses.)27 b(If)16 b(pro)q(cessing)75 1755 y(is)d(required)i(on)e(another)h(pro)q(cess,)h (this)e(should)g(b)q(e)i(transparen)o(t)f(to)f(the)i(application)d (\(hence)j(the)f(need)g(for)f(an)75 1811 y(in)o(terupt)h(and)g(an)g(in) o(terupt)g(handler\).)75 2037 y Ft(3.10)59 b(P)n(ersistent)19 b(communication)f(objects)75 2139 y FC(Often)k(a)g(comm)o(unication)h (with)f(the)g(same)g(argumen)o(t)f(list)i(is)g(rep)q(eatedly)g (executed)g(within)g(the)75 2195 y(inner)c(lo)q(op)g(of)e(a)h(parallel) i(computation.)28 b(In)18 b(suc)o(h)h(a)f(situation,)g(it)h(ma)o(y)e(b) q(e)i(p)q(ossible)g(to)f(optimize)75 2252 y(the)h(comm)o(unication)g(b) o(y)g(binding)h(the)f(list)g(of)g(comm)o(unication)g(argumen)o(ts)f(to) g(a)g(comm)o(unication)75 2308 y(request)11 b(once)g(and,)g(then,)h (rep)q(eatedly)g(using)g(the)f(comm)o(unication)g(request)g(to)f (initiate)i(and)f(complete)75 2365 y(messages.)19 b(The)d(comm)o (unication)f(request)h(th)o(us)f(created)g(can)g(b)q(e)h(though)o(t)e (of)h(as)f(a)h(comm)o(unication)75 2421 y(p)q(ort)c(or)f(a)h(\\half-c)o (hannel")h(.)18 b(It)12 b(do)q(es)f(not)f(pro)o(vide)i(the)f(full)h (functionalit)o(y)h(of)d(a)h(con)o(v)o(en)o(tional)g(c)o(hannel,)75 2478 y(since)i(there)g(is)g(no)f(binding)i(of)e(the)g(send)h(p)q(ort)f (to)f(the)i(receiv)o(e)g(p)q(ort:)18 b(this)13 b(construct)f(allo)o(ws) g(reduction)75 2534 y(of)18 b(the)g(o)o(v)o(erhead)f(for)h(comm)o (unication)g(b)q(et)o(w)o(een)h(pro)q(cessor)e(and)h(comm)o(unication)h (con)o(troller,)g(but)75 2591 y(not)c(the)g(o)o(v)o(erhead)g(for)f (comm)o(unication)i(b)q(et)o(w)o(een)g(one)f(comm)o(unication)h(con)o (troller)g(and)f(another.)166 2647 y(A)i(comm)o(unication)g(request)g (is)g(created)g(using)h(one)f(of)f(the)h(four)f(follo)o(wing)i(calls.) 26 b(These)17 b(calls)75 2704 y(in)o(v)o(olv)o(e)f(no)f(comm)o (unication.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 47 39 46 bop 75 -100 a Fs(3.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (OBJECTS)673 b FC(39)166 45 y(The)10 b(function)i Fv(MPI)p 513 45 14 2 v 15 w(CREA)l(TE)p 697 45 V 18 w(SEND)f FC(creates)f(a)g (comm)o(unication)h(request)f(for)g(a)g(standard)g(mo)q(de)75 102 y(send)16 b(op)q(eration,)f(and)g(binds)i(to)d(it)i(all)g(the)f (argumen)o(ts)f(of)h(a)g(send)h(op)q(eration.)75 214 y Fv(MPI)p 160 214 V 16 w(CREA)l(TE)p 345 214 V 17 w(SEND\(buf,)g (count,)g(datat)o(yp)q(e,)h(dest,)f(tag,)f(comm,)e(request\))117 300 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 392 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 484 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 577 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o(teger\))117 669 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 761 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 854 y(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 987 y Fo(int)23 b(MPI)p 245 987 15 2 v 17 w(Create)p 406 987 V 17 w(send\(void*)f(buf,)h(int)h (count,)f(MPI)p 1139 987 V 17 w(Datatype)f(datatype,)h(int)h(dest,)393 1043 y(int)g(tag,)f(MPI)p 683 1043 V 17 w(Comm)g(comm,)g(MPI)p 1034 1043 V 17 w(Comm)p 1147 1043 V 17 w(request)g(request\))75 1138 y(MPI)p 150 1138 V 17 w(CREATE)p 311 1138 V 16 w(SEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g(REQUEST,)g(IERROR\))170 1195 y()g(BUF\(*\))170 1251 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 1346 y FC(The)16 b(function)h Fv(MPI)p 524 1346 14 2 v 16 w(CREA)l(TE)p 709 1346 V 17 w(RSEND)g FC(creates)e(a)h(comm)o (unication)h(ob)s(ject)e(for)g(a)h(ready)g(mo)q(de)75 1403 y(send)g(op)q(eration.)75 1515 y Fv(MPI)p 160 1515 V 16 w(CREA)l(TE)p 345 1515 V 17 w(RSEND\(buf,)g(count,)g(datat)o(yp)q (e,)h(dest,)f(tag,)f(comm,)e(request\))117 1601 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 1693 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 1785 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 1878 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1970 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 2062 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 2155 y(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 2288 y Fo(int)23 b(MPI)p 245 2288 15 2 v 17 w(Create)p 406 2288 V 17 w(rsend\(void*)f(buf,)h(int)h (count,)f(MPI)p 1163 2288 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 2344 y(int)h(tag,)f(MPI)p 683 2344 V 17 w(Comm)g(comm,)g(MPI)p 1034 2344 V 17 w(Comm)p 1147 2344 V 17 w(request)g(request\))75 2439 y(MPI)p 150 2439 V 17 w(CREATE)p 311 2439 V 16 w(RSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 2496 y()g(BUF\(*\))170 2552 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)166 2647 y FC(The)15 b(function)i Fv(MPI)p 523 2647 14 2 v 15 w(CREA)l(TE)p 707 2647 V 18 w(SSEND)f FC(creates)f(a)g(comm)o(unication)h(ob)s(ject)e (for)h(a)g(sync)o(hronous)75 2704 y(mo)q(de)g(send)h(op)q(eration.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 48 40 47 bop 75 -100 a FC(40)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(CREA)l(TE)p 345 45 V 17 w(SSEND\(buf,)g(count,)g (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm,)e(request\))117 122 y Fq(IN)155 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 198 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o(teger\))117 424 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 499 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 575 y(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 699 y Fo(int)23 b(MPI)p 245 699 15 2 v 17 w(Create)p 406 699 V 17 w(ssend\(void*)f(buf,)h(int)h (count,)f(MPI)p 1163 699 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 756 y(int)h(tag,)f(MPI)p 683 756 V 17 w(Comm)g(comm,)g(MPI)p 1034 756 V 17 w(Comm)p 1147 756 V 17 w(request)g(request\))75 842 y(MPI)p 150 842 V 17 w(CREATE)p 311 842 V 16 w(SSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 899 y()g(BUF\(*\))170 955 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,) g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)166 1042 y FC(The)15 b(function)h Fv(MPI)p 522 1042 14 2 v 15 w(CREA)l(TE)p 706 1042 V 18 w(RECV)g FC(creates)e(a)h(comm)o(unication)g(ob)s(ject)f (for)h(a)f(receiv)o(e)i(op)q(era-)75 1098 y(tion.)75 1202 y Fv(MPI)p 160 1202 V 16 w(CREA)l(TE)p 345 1202 V 17 w(RECV\(buf,)g(count,)h(datat)o(yp)q(e,)f(source,)f(tag,)g(comm,)e (request\))117 1279 y Fq(OUT)108 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1355 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(receiv)o(ed)j(\(in)o(teger\))117 1430 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 1505 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(source)j(or)d(MPI)p 1309 1505 13 2 v 16 w(ANY)p 1418 1505 V 15 w(SOUR)o(CE)h(\(in)o (teger\))117 1581 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(or)h(MPI)p 1270 1581 V 15 w(ANY)p 1378 1581 V 16 w(T)m(A)o(G)f(\(in)o(teger\))117 1656 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 1731 y(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 1856 y Fo(int)23 b(MPI)p 245 1856 15 2 v 17 w(Create)p 406 1856 V 17 w(recv\(void*)f(buf,)h(int)h(count,)f(MPI)p 1139 1856 V 17 w(Datatype)f(datatype,)h(int)h(source,)393 1912 y(int)g(tag,)f(MPI)p 683 1912 V 17 w(Comm)g(comm,)g(MPI)p 1034 1912 V 17 w(Comm)p 1147 1912 V 17 w(request)g(request\))75 1999 y(MPI)p 150 1999 V 17 w(CREATE)p 311 1999 V 16 w(RECV\(BUF,)g (COUNT,)g(DATATYPE,)f(SOURCE,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 2055 y()g(BUF\(*\))170 2112 y(INTEGER)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 2198 y FC(A)18 b(comm)o(unication)g(\(send)g(or)g(receiv)o(e\))g(that)f (uses)h(a)g(prede\014ned)h(request)f(is)h(initiated)g(b)o(y)f(the)75 2255 y(function)e Fv(MPI)p 338 2255 14 2 v 16 w(ST)l(ART)p FC(.)75 2359 y Fv(MPI)p 160 2359 V 16 w(ST)l(ART\(request\))117 2436 y Fq(INOUT)62 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 2561 y Fo(int)23 b(MPI)p 245 2561 15 2 v 17 w(Start\(MPI)p 478 2561 V 16 w(Comm)p 590 2561 V 17 w(request)g(request\))75 2647 y(MPI)p 150 2647 V 17 w(START\(REQUEST,)f(IERROR\))170 2704 y(INTEGER)h(REQUEST,)g (IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 49 41 48 bop 75 -100 a Fs(3.10.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (OBJECTS)673 b FC(41)166 45 y(The)11 b(comm)o(unication)h(ob)s(ject)f (asso)q(ciated)g(with)h Fv(request)h FC(should)f(b)q(e)g(an)f(ob)s (ject)g(that)g(w)o(as)f(created)75 102 y(b)o(y)k(one)g(of)g(the)g (previous)h(four)e(functions,)i(so)f(that)f(all)i(the)f(comm)o (unication)h(argumen)o(ts)e(are)h(already)75 158 y(de\014ned.)23 b(A)16 b(send)g(can)g(b)q(e)h(started)e(pro)o(vided)h(that)f(the)h (previous)h(send)f(using)h(the)e(same)h(ob)s(ject)f(has)75 214 y(completed,)d(or)f(as)f(so)q(on)h(as)f(the)h(ob)s(ject)f(is)i (created,)f(if)g(it)g(has)g(not)g(y)o(et)f(b)q(een)i(used)f(in)h(a)f (comm)o(unication.)75 271 y(In)j(addition,)g(if)g(the)g(comm)o (unication)g(mo)q(de)f(is)h Fo(ready)f FC(then)g(a)g(matc)o(hing)h (receiv)o(e)g(should)g(b)q(e)g(p)q(osted.)75 327 y(The)k(send)h (bu\013er)f(should)g(not)g(b)q(e)h(accessed)f(after)f(the)h(send)h(is)f (started,)g(and)g(un)o(til)h(the)f(op)q(eration)75 384 y(completes.)166 443 y(A)c(receiv)o(e)g(can)g(b)q(e)g(started)f(pro)o (vided)i(that)e(the)h(preceding)h(receiv)o(e)f(using)h(the)f(same)f(ob) s(ject)g(has)75 500 y(completed,)f(or)f(as)f(so)q(on)h(as)f(the)h(ob)s (ject)f(is)i(created,)f(if)g(it)g(has)g(not)g(y)o(et)f(b)q(een)i(used)f (in)h(a)f(comm)o(unication.)75 556 y(The)18 b(receiv)o(e)h(bu\013er)f (should)h(not)e(b)q(e)i(accessed)g(after)e(the)h(receiv)o(e)h(is)f (started,)g(un)o(til)h(the)f(op)q(eration)75 613 y(completes.)166 672 y(The)23 b(call)g(is)g(non)o(blo)q(c)o(king,)i(with)e(similar)h (seman)o(tics)e(to)g(the)g(non)o(blo)q(c)o(king)i(comm)o(unication)75 728 y(op)q(erations)15 b(describ)q(ed)i(in)g(section)e(3.8.)75 835 y Fv(MPI)p 160 835 14 2 v 16 w(ST)l(ART)l(ALL\(count,)h(a)o(rra)o (y)p 641 835 V 15 w(of)p 693 835 V 16 w(requests\))117 915 y Fq(IN)155 b Fv(count)482 b Fq(list)14 b(length)f(\(in)o(teger\)) 117 996 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 996 V 15 w(of)p 468 996 V 16 w(requests)272 b Fq(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handle\))75 1123 y Fo(int)23 b(MPI)p 245 1123 15 2 v 17 w(Startall\(int)f(count,)h(MPI)p 811 1123 V 17 w(Comm)p 924 1123 V 17 w(request)g(*array)p 1276 1123 V 16 w(of)p 1340 1123 V 17 w(requests\))75 1213 y(MPI)p 150 1213 V 17 w(STARTALL\(COUNT,)f(ARRAY)p 669 1213 V 16 w(OF)p 733 1213 V 17 w(REQUESTS,)h(IERROR\))170 1269 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1269 V 17 w(OF)p 716 1269 V 17 w(REQUESTS\(*\),)f(IERROR)166 1358 y FC(Start)i(all)j(comm)o (unications)f(asso)q(ciated)g(with)f(requests)h(in)g Fv(a)o(rra)o(y)p 1399 1358 14 2 v 15 w(of)p 1451 1358 V 16 w(requests)p FC(.)52 b(A)26 b(call)h(to)75 1415 y Fv(MPI)p 160 1415 V 16 w(ST)l(ART)l(ALL\(count,)76 b(a)o(rra)o(y)p 701 1415 V 14 w(of)p 752 1415 V 16 w(requests\))65 b FC(has)d(the)h(same)f(e\013ect)h(as)f(calls)i(to)75 1471 y Fv(MPI)p 160 1471 V 16 w(ST)l(ART\(a)o(rra)o(y)p 431 1471 V 15 w(of)p 483 1471 V 16 w(requests[i]\))p FC(,)23 b(executed)d(for)g Fo(i=0)j(,...,)g(count-1)p FC(,)d(in)h(some)e(arbitrary)g(or-)75 1528 y(der.)166 1587 y(A)e(comm)o(unication)g(started)f(with)h(a)g(call)h(to)e Fv(MPI)p 1063 1587 V 16 w(ST)l(ART)h FC(or)f Fv(MPI)p 1377 1587 V 16 w(ST)l(ART)l(ALL)i FC(is)f(completed)75 1643 y(b)o(y)11 b(a)h(call)g(to)f Fv(MPI)p 385 1643 V 16 w(W)l(AIT)p FC(,)g Fv(MPI)p 620 1643 V 16 w(TEST)p FC(,)h(or)f(one)g(of)g(the)h(deriv)o(ed)h(functions)f(describ)q(ed)h (in)g(section)f(3.8.5.)75 1700 y(These)k(comm)o(unication)h(completion) g(functions)g(do)f(not)f(deallo)q(cate)j(the)e(request,)g(and)g(this)g (can)g(b)q(e)75 1756 y(reused)g(anew)g(b)o(y)f(a)h Fv(MPI)p 521 1756 V 15 w(ST)l(ART)h FC(call.)22 b(The)16 b(ob)s(ject)f(needs)h (to)f(b)q(e)i(explicitly)h(deallo)q(cated)f(b)o(y)f(a)f(call)75 1813 y(to)j(the)g(function)i Fv(MPI)p 482 1813 V 16 w(REQUEST)p 701 1813 V 17 w(FREE)p FC(,)f(b)q(elo)o(w.)30 b(W)l(e)19 b(th)o(us)f(ha)o(v)o(e)g(t)o(w)o(o)g(t)o(yp)q(es)g(of)g(comm)o (unication)75 1869 y(ob)s(jects:)28 b Fx(p)q(ersisten)o(t)19 b FC(ob)s(jects,)g(whic)o(h)h(are)g(allo)q(cated)g(b)o(y)f(a)g(call)i (to)e Fv(MPI)p 1427 1869 V 15 w(CREA)l(TE)p 1611 1869 V 18 w(xxx)p FC(,)h(and)f(are)75 1926 y(explicitly)c(deallo)q(cated)e (b)o(y)f Fv(MPI)p 649 1926 V 16 w(REQUEST)p 868 1926 V 18 w(FREE)p FC(,)g(and)h Fx(ephemeral)f FC(ob)s(jects)f(that)h(p)q (ersist)g(for)g(one)75 1982 y(comm)o(unication)i(only:)20 b(they)14 b(are)f(created)h(b)o(y)f(a)h(non)o(blo)q(c)o(king)h(comm)o (unication)f(initiation)h(function,)75 2039 y(and)g(are)g(freed)h(b)o (y)f(the)g(comm)o(unication)h(completion)g(call.)75 2145 y Fv(MPI)p 160 2145 V 16 w(REQUEST)p 379 2145 V 18 w(FREE\(request\)) 117 2225 y Fq(INOUT)62 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 2353 y Fo(int)23 b(MPI)p 245 2353 15 2 v 17 w(Request)p 430 2353 V 16 w(free\(MPI)p 638 2353 V 17 w(Comm)p 751 2353 V 16 w(request)g(request\))75 2442 y(MPI)p 150 2442 V 17 w(REQUEST)p 335 2442 V 16 w(FREE\(REQUEST,)f(IERROR\))170 2498 y(INTEGER)h(REQUEST,)g(IERROR)166 2588 y FC(deallo)q(cated)15 b(when)f(there)g(are)g(no)g(p)q(ending)h (comm)o(unications)g(in)o(v)o(olving)g(this)f(ob)s(ject,)f(at)g(whic)o (h)75 2644 y(p)q(oin)o(t)j(the)f(request)g(b)q(ecomes)h(n)o(ull.)166 2704 y(The)j(call)g(is)g(non)o(blo)q(c)o(king.)31 b(It)18 b(is)h(p)q(ermissible)i(to)d(call)h Fv(MPI)p 1249 2704 14 2 v 16 w(REQUEST)p 1468 2704 V 18 w(FREE\(request\))h FC(after)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 42 50 42 49 bop 75 -100 a FC(42)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FC(a)j(comm)o (unication)g(that)f(uses)h Fv(request)i FC(has)e(b)q(een)h(initiated,)h (but)e(b)q(efore)g(it)g(has)g(completed.)32 b(The)75 102 y(ob)s(ject)15 b(will)j(b)q(e)f(deallo)q(cated)g(after)e(the)h (comm)o(unication)h(completes.)22 b(It)16 b(is)h(erroneous)f(to)f (initiate)i(a)75 158 y(comm)o(unication)f(with)f(a)g(request)h(that)e (has)h(b)q(een)h(freed.)166 214 y(A)f(correct)f(in)o(v)o(o)q(cation)i (of)e(the)h(functions)h(describ)q(ed)g(in)g(this)f(section)h(will)g(o)q (ccur)g(in)f(a)g(sequence)75 271 y(of)g(the)g(form)166 327 y Fx(Create)i FC(\()p Fx(Start)h(Complete)p FC(\))725 311 y Fj(\003)762 327 y Fx(F)l(ree)e Fw(;)166 384 y FC(where)g FA(\003)g FC(indicates)i(zero)e(or)f(more)h(rep)q(etitions.)24 b(If)16 b(the)g(same)g(comm)o(unication)h(ob)s(ject)f(is)g(used)75 440 y(in)k(sev)o(eral)f(concurren)o(t)g(threads,)g(it)g(is)h(the)f (user)g(resp)q(onsibilit)o(y)i(to)e(co)q(ordinate)g(calls)h(so)f(that)f (the)75 497 y(correct)d(sequence)h(is)g(ob)q(ey)o(ed.)166 553 y(A)c(send)g(op)q(eration)f(initiated)j(with)d Fv(MPI)p 874 553 14 2 v 16 w(ST)l(ART)i FC(can)e(b)q(e)i(matc)o(hed)e(with)h(an) o(y)f(receiv)o(e)i(op)q(eration)75 610 y(and,)g(lik)o(ewise,)h(a)e (receiv)o(e)i(op)q(eration)e(initiated)i(with)f Fv(MPI)p 1098 610 V 16 w(ST)l(ART)g FC(can)g(receiv)o(e)g(messages)f(generated) 75 666 y(b)o(y)j(an)o(y)g(send)h(op)q(eration.)75 808 y Ft(3.11)59 b(Send-receive)75 909 y FC(The)18 b Fx(send-receiv)o(e)g FC(op)q(erations)g(com)o(bine)h(in)g(one)f(call)h(the)f(sending)i(of)d (a)h(message)g(to)f(one)h(desti-)75 966 y(nation)e(and)g(the)g (receiving)i(of)d(another)h(message,)f(from)g(another)h(destination,)g (p)q(ossibly)i(the)e(same.)75 1022 y(A)k(send-receiv)o(e)j(op)q (eration)d(is)h(v)o(ery)f(useful)i(for)d(executing)j(a)e(shift)h(op)q (eration)f(across)g(a)g(c)o(hain)h(of)75 1079 y(pro)q(cesses.)28 b(If)18 b(blo)q(c)o(king)h(sends)g(and)f(receiv)o(es)h(are)e(used)h (for)g(suc)o(h)g(shift,)g(then)g(one)g(needs)h(to)e(order)75 1135 y(the)g(sends)g(and)g(receiv)o(es)h(correctly)f(\(e.g.,)f(ev)o(en) h(pro)q(cesses)g(send,)h(next)f(receiv)o(e,)g(o)q(dd)h(pro)q(cesses)f (re-)75 1192 y(ceiv)o(e)g(\014rst,)f(next)g(send\))h(so)f(as)f(to)h (prev)o(en)o(t)g(cyclic)i(dep)q(endencies)i(that)15 b(lead)i(to)f (deadlo)q(c)o(k.)24 b(When)16 b(a)75 1248 y(send-receiv)o(e)e(op)q (eration)e(is)g(used,)h(the)f(comm)o(unication)h(subsystem)f(tak)o(es)f (care)g(of)h(these)g(issues.)20 b(The)75 1305 y(send-receiv)o(e)14 b(op)q(eration)f(can)f(b)q(e)h(used)g(in)h(conjunction)f(with)g(the)f (functions)i(describ)q(ed)g(in)f(Chapter)f(6)75 1361 y(in)19 b(order)f(to)g(p)q(erform)g(shifts)h(on)f(v)m(arious)h(logical) h(top)q(ologies.)30 b(Also,)19 b(a)f(send-receiv)o(e)i(op)q(eration)e (is)75 1418 y(useful)e(for)f(implemen)o(ting)i(remote)e(pro)q(cedure)h (calls.)166 1474 y(A)g(message)f(sen)o(t)g(b)o(y)h(a)f(send-receiv)o(e) j(op)q(eration)d(can)h(b)q(e)h(receiv)o(ed)f(b)o(y)g(a)f(regular)h (receiv)o(e)h(op)q(er-)75 1530 y(ation,)e(and)g(vice)h(v)o(ersa.)75 1634 y Fv(MPI)p 160 1634 V 16 w(SENDRECV\(sendbuf,)24 b(sendcount,)f(sendt)o(yp)q(e,)g(dest,)f(sendtag,)g(recvbuf,)g (recvcount,)f(recvt)o(yp)q(e,)75 1691 y(source,)15 b(recvtag,)g(comm,)e (status\))117 1768 y Fq(IN)155 b Fv(sendbuf)444 b Fq(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1842 y(IN)155 b Fv(sendcount)399 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1915 y(IN)155 b Fv(sendt)o(yp)q(e)424 b Fq(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(bu\013er)g(\(handle\))117 1989 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2063 y(IN)155 b Fv(sendtag)443 b Fq(send)15 b(tag)e(\(in)o(teger\))117 2137 y(OUT)108 b Fv(recvbuf)450 b Fq(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\)) 117 2210 y(IN)155 b Fv(recvcount)405 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 2284 y(IN)155 b Fv(recvt)o(yp)q(e)430 b Fq(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(receiv)o(e)i(bu\013er)g(\(handle\))117 2358 y(IN)155 b Fv(source)468 b Fq(rank)14 b(of)f(source)j(\(in)o (teger\))117 2432 y(IN)155 b Fv(recvtag)449 b Fq(receiv)o(e)15 b(tag)f(\(in)o(teger\))117 2505 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 2579 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\))75 2704 y Fo(int)23 b(MPI)p 245 2704 15 2 v 17 w(Sendrecv\(void)f (*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 2704 V 17 w(Datatype)f(sendtype,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 43 51 43 50 bop 75 -100 a Fs(3.11.)29 b(SEND-RECEIVE)1282 b FC(43)393 45 y Fo(int)24 b(dest,)f(int)g(sendtag,)g(void)g(*recvbuf,)g (int)g(recvcount,)393 102 y(MPI)p 468 102 15 2 v 17 w(Datatype)g (recvtype,)f(int)i(source,)f(MPI)p 1297 102 V 16 w(Datatype)g(recvtag,) 393 158 y(MPI)p 468 158 V 17 w(Comm)g(comm,)g(MPI)p 819 158 V 17 w(Status)g(*status\))75 245 y(MPI)p 150 245 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(DEST,)i(SENDTAG,)e (RECVBUF,)393 301 y(RECVCOUNT,)h(RECVTYPE,)f(SOURCE,)h(RECVTAG,)g (COMM,)g(STATUS,)g(IERROR\))170 357 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 414 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(DEST,)i (SENDTAG,)e(RECVCOUNT,)h(RECVTYPE,)170 470 y(SOURCE,)g(RECVTAG,)g (COMM,)g(STATUS\(MPI)p 962 470 V 16 w(STATUS)p 1122 470 V 16 w(SIZE\),)h(IERROR)166 557 y FC(Execute)19 b(a)e(blo)q(c)o(king)j (send)f(and)f(receiv)o(e)h(op)q(eration.)29 b(Both)18 b(send)h(and)f(receiv)o(e)h(use)g(the)f(same)75 613 y(comm)o(unicator,) c(but)g(p)q(ossibly)i(di\013eren)o(t)f(tags.)j(The)d(send)g(bu\013er)f (and)h(receiv)o(e)g(bu\013ers)g(are)f(disjoin)o(t,)75 670 y(and)h(ma)o(y)g(ha)o(v)o(e)g(di\013eren)o(t)g(lengths)h(and)f (datat)o(yp)q(es.)75 773 y Fv(MPI)p 160 773 14 2 v 16 w(SENDRECV)p 408 773 V 17 w(REPLA)o(CE\(buf,)k(count,)f(datat)o(yp)q (e,)h(dest,)f(sendtag,)h(source,)f(recvtag,)f(comm,)e(sta-)75 830 y(tus\))117 907 y Fq(INOUT)62 b Fv(buf)527 b Fq(initial)12 b(address)j(of)f(send)g(and)g(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 982 y(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)9 b(of)g(elemen)o(ts)h (in)f(send)i(and)e(receiv)o(e)i(bu\013er)g(\(in)o(teger\))117 1058 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(and)f(receiv)o(e)h(bu\013er)g (\(handle\))117 1133 y(IN)155 b Fv(dest)511 b Fq(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1208 y(IN)155 b Fv(sendtag)443 b Fq(send)15 b(message)f(tag)f(\(in)o(teger\))117 1283 y(IN)155 b Fv(source)468 b Fq(rank)14 b(of)f(source)j(\(in)o (teger\))117 1359 y(IN)155 b Fv(recvtag)449 b Fq(receiv)o(e)15 b(message)f(tag)f(\(in)o(teger\))117 1434 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 1509 y(OUT)108 b Fv(status)476 b Fq(status)15 b(ob)r(ject)g(\(Status\)) 75 1634 y Fo(int)23 b(MPI)p 245 1634 15 2 v 17 w(Sendrecv)p 454 1634 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 1634 V 17 w(Datatype)g(datatype,)393 1690 y(int)h(dest,)f(int)g (sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 1690 V 16 w(Comm)h(comm,)393 1746 y(MPI)p 468 1746 V 17 w(Status)f(*status\))75 1833 y(MPI)p 150 1833 V 17 w(SENDRECV)p 359 1833 V 16 w(REPLACE\(BUF,)f(COUNT,)h(DATATYPE,)g (DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)393 1889 y(COMM,)g(STATUS,)g (IERROR\))170 1946 y()g(BUF\(*\))170 2002 y(INTEGER)g(COUNT,)g (DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)g(COMM,)170 2059 y(STATUS\(MPI)p 413 2059 V 16 w(STATUS)p 573 2059 V 17 w(SIZE\),)g(IERROR)166 2145 y FC(Execute)15 b(a)f(blo)q(c)o(king)i (send)f(and)g(receiv)o(e;)g(the)g(same)f(bu\013er)g(is)h(used)h(b)q (oth)e(for)g(the)h(send)g(and)g(for)75 2202 y(the)g(receiv)o(e,)h(so)f (that)f(the)i(message)e(sen)o(t)h(is)h(replaced)g(b)o(y)g(the)f (message)g(receiv)o(ed.)166 2258 y(The)f(seman)o(tics)g(of)f(a)h (send-receiv)o(e)h(op)q(eration)f(is)g(what)g(w)o(ould)g(obtain)g(if)g (the)g(caller)h(fork)o(ed)e(t)o(w)o(o)75 2315 y(concurren)o(t)19 b(threads,)f(one)h(to)f(execute)h(the)g(send,)g(and)g(one)g(to)e (execute)j(the)e(receiv)o(e,)i(follo)o(w)o(ed)f(b)o(y)75 2371 y(a)e(join)g(of)g(these)g(t)o(w)o(o)f(threads.)25 b(Additional)19 b(in)o(termediate)f(bu\013ering)g(is)f(needed)i(for)d (the)i(\\replace")75 2428 y(v)m(arian)o(t.)166 2484 y(A)11 b(send-receiv)o(e)i(cannot)f(b)q(e)g(implemen)o(ted)h(b)o(y)e(a)g(blo)q (c)o(king)i(send)f(follo)o(w)o(ed)g(b)o(y)f(a)g(blo)q(c)o(king)i (receiv)o(e)75 2541 y(or)i(a)g(blo)q(c)o(king)i(receiv)o(e,)f(follo)o (w)o(ed)g(b)o(y)g(a)f(blo)q(c)o(king)i(send.)22 b(Consider)16 b(the)f(follo)o(wing)i(co)q(de,)e(where)h(t)o(w)o(o)75 2597 y(pro)q(cesses)g(exc)o(hange)f(messages:)75 2704 y Fo(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 44 52 44 51 bop 75 -100 a FC(44)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(IF)24 b(\(rank.EQ.0\))123 102 y(THEN)170 158 y(CALL)g (MPI_SENDRECV\(send_buff,)c(send_count,)j(MPI_REAL,)f(0,)i(send_tag,) 242 214 y(recv_buff,)f(recv_count,)f(MPI_REAL,)h(1,)g(recv_tag,)g (comm,)g(status,)g(ierr\))123 271 y(ELSE)170 327 y(CALL)h (MPI_SENDRECV\(send_buff,)c(send_count,)j(MPI_REAL,)f(1,)i(send_tag,) 242 384 y(recv_buff,)f(recv_count,)f(MPI_REAL,)h(0,)g(tag,)g(comm,)h (status,)f(ierr\))75 440 y(END)g(IF)166 553 y FC(If)16 b(the)f(send)h(receiv)o(es)h(are)e(replaced)i(either)f(b)o(y)f(blo)q(c) o(king)i(send,)f(follo)o(w)o(ed)g(b)o(y)g(blo)q(c)o(king)g(receiv)o(e,) 75 610 y(or)g(blo)q(c)o(king)h(receiv)o(e,)g(follo)o(w)o(ed)g(b)o(y)f (blo)q(c)o(king)i(send,)f(then)f(the)h(co)q(de)g(ma)o(y)e(deadlo)q(c)o (k.)24 b(On)17 b(the)f(other)75 666 y(hand,)f(send-receiv)o(e)i(can)f (b)q(e)f(implemen)o(ted)i(using)f(non)o(blo)q(c)o(king)h(sends)e(and)h (receiv)o(es.)75 817 y Ft(3.12)59 b(Null)21 b(p)n(ro)r(cesses)75 921 y FC(In)15 b(man)o(y)e(instances,)i(it)f(is)h(con)o(v)o(enien)o(t)g (to)e(sp)q(ecify)j(a)d(\\dumm)o(y")h(source)g(or)g(destination)h(for)e (comm)o(u-)75 978 y(nication.)29 b(This)18 b(simpli\014es)i(the)e(co)q (de)g(that)f(is)h(needed)i(for)d(dealing)i(with)f(b)q(oundaries,)h (e.g.,)e(in)i(the)75 1034 y(case)c(of)g(a)g(non-circular)i(shift)e (done)h(with)f(calls)i(to)d(send-receiv)o(e.)166 1092 y(The)f(sp)q(ecial)j(v)m(alue)26 b Fl(MPI)p 608 1092 13 2 v 15 w(PROCNULL)12 b FC(can)i(b)q(e)g(used)g(instead)g(of)f(a)g (rank)g(wherev)o(er)g(a)g(source)g(or)g(a)75 1149 y(destination)j (argumen)o(t)f(is)g(required)i(in)f(a)f(call.)21 b(A)15 b(comm)o(unication)h(with)g(pro)q(cess)29 b Fl(MPI)p 1649 1149 V 15 w(PROCNULL)75 1205 y FC(has)13 b(no)h(e\013ect:)k(a)13 b(send)h(to)25 b Fl(MPI)p 633 1205 V 14 w(PROCNULL)13 b FC(succeeds)i(and)e(returns)g(as)g(so)q(on)h(as)f(p)q(ossible.)21 b(A)13 b(receiv)o(e)75 1262 y(from)26 b Fl(MPI)p 271 1262 V 14 w(PROCNULL)13 b FC(succeeds)i(and)f(returns)g(as)f(so)q(on)h (as)f(p)q(ossible)j(with)e(no)g(mo)q(di\014cations)h(to)e(the)75 1318 y(receiv)o(e)18 b(bu\013er.)25 b(When)17 b(a)g(receiv)o(e)h(with)f Fv(source)h FC(=)32 b Fl(MPI)p 1090 1318 V 15 w(PROCNULL)16 b FC(is)i(executed)f(then)h(the)f(status)75 1374 y(ob)s(ject)e(returns) 30 b Fv(source)16 b FC(=)29 b Fl(MPI)p 661 1374 V 14 w(PROCNULL)p FC(,)14 b Fv(tag)h FC(=)30 b Fl(MPI)p 1132 1374 V 14 w(ANY)p 1231 1374 V 15 w(T)m(A)o(G)14 b FC(and)h Fv(count=0)p FC(.)75 1525 y Ft(3.13)59 b(Derived)19 b(datat)n(yp)r(es) 75 1630 y FC(Up)c(to)f(no)o(w,)g(all)i(p)q(oin)o(t)f(to)f(p)q(oin)o(t)h (comm)o(unication)h(in)o(v)o(olv)o(ed)f(only)g(con)o(tiguous)g (bu\013ers)g(con)o(taining)g(a)75 1686 y(sequence)g(of)e(elemen)o(ts)h (of)f(the)g(same)g(t)o(yp)q(e.)19 b(This)14 b(is)g(to)q(o)f (constraining)h(on)f(t)o(w)o(o)f(accoun)o(ts:)19 b(One)14 b(often)75 1742 y(w)o(an)o(ts)f(to)h(pass)g(messages)g(that)f(con)o (tain)i(v)m(alues)g(with)g(di\013eren)o(t)f(datat)o(yp)q(es)g(\(e.g.,)f (an)h(in)o(teger)h(coun)o(t,)75 1799 y(follo)o(w)o(ed)i(b)o(y)f(a)g (sequence)i(of)e(real)h(n)o(um)o(b)q(ers\);)f(and)h(one)g(often)f(w)o (an)o(ts)f(to)h(send)h(noncon)o(tiguous)g(data)75 1855 y(\(e.g.)h(a)12 b(sub-blo)q(c)o(k)i(of)e(a)g(matrix\).)18 b(One)13 b(solution)g(is)g(to)f(pro)o(vide)h(functions)g(that)e(pac)o (k)i(noncon)o(tiguous)75 1912 y(data)f(in)o(to)h(a)g(con)o(tiguous)g (bu\013er)g(at)f(the)h(sender)h(site)f(and)g(unpac)o(k)h(it)f(bac)o(k)g (at)f(the)h(receiv)o(er)h(site.)19 b(This)75 1968 y(has)f(the)g(disadv) m(an)o(tage)g(of)g(requiring)h(additional)h(memory-to-memory)c(cop)o(y) i(op)q(erations)g(at)g(b)q(oth)75 2025 y(sites,)12 b(ev)o(en)f(when)g (the)g(comm)o(unication)g(subsystem)g(has)f(scatter-gather)g (capabilities.)20 b(Instead,)12 b(MPI)75 2081 y(pro)o(vides)22 b(mec)o(hanisms)f(to)g(sp)q(ecify)h(more)f(general,)i(mixed)f(and)f (noncon)o(tiguous)g(comm)o(unication)75 2138 y(bu\013ers.)29 b(It)19 b(is)g(up)g(to)f(the)g(implemen)o(tation)i(to)e(decide)i (whether)e(data)g(should)h(b)q(e)h(\014rst)e(pac)o(k)o(ed)g(in)75 2194 y(a)e(con)o(tiguous)h(bu\013er)g(b)q(efore)g(b)q(eing)h (transmitted,)f(or)f(whether)h(it)g(can)g(b)q(e)g(collected)i(directly) f(from)75 2251 y(where)d(it)h(resides.)166 2308 y(The)e(general)h(mec)o (hanisms)g(pro)o(vided)g(here)f(allo)o(w)h(one)f(to)g(transfer)f (directly)l(,)j(without)e(cop)o(ying,)75 2365 y(ob)s(jects)j(of)g(v)m (arious)h(shap)q(e)h(and)f(size.)28 b(It)17 b(is)i(not)e(assumed)h (that)f(the)g(MPI)h(library)g(is)h(cognizan)o(t)e(of)75 2421 y(the)f(ob)s(jects)g(declared)h(in)g(the)f(host)g(language;)g(th)o (us,)g(if)g(one)g(w)o(an)o(ts)f(to)h(transfer)f(a)h(structure,)f(or)h (an)75 2478 y(arra)o(y)d(section,)i(it)f(will)i(b)q(e)f(necessary)f(to) g(pro)o(vide)h(in)g(MPI)f(a)g(de\014nition)i(of)e(a)g(comm)o(unication) h(bu\013er)75 2534 y(that)e(mimics)j(the)e(de\014nition)i(of)e(the)g (structure)g(or)f(arra)o(y)g(section)i(in)g(question.)20 b(These)15 b(facilities)h(can)75 2591 y(b)q(e)j(used)g(b)o(y)f(library) h(designers)h(to)d(de\014ne)j(comm)o(unication)f(functions)g(that)f (can)g(transfer)g(ob)s(jects)75 2647 y(de\014ned)g(in)f(the)f(host)g (language)g({)g(b)o(y)h(deco)q(ding)g(their)g(de\014nitions)h(as)e(a)o (v)m(ailable)i(in)f(a)f(sym)o(b)q(ol)h(table)75 2704 y(or)e(a)g(dop)q(e)g(v)o(ector.)20 b(Suc)o(h)15 b(higher-lev)o(el)j (comm)o(unication)e(functions)g(are)f(not)g(part)f(of)h(MPI.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 45 53 45 52 bop 75 -100 a Fs(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(45)166 45 y(More)12 b(general)g(comm)o(unication)i(bu\013ers) e(are)g(sp)q(eci\014ed)i(b)o(y)f(replacing)g(the)g(basic)g(datat)o(yp)q (es)f(that)75 102 y(ha)o(v)o(e)e(b)q(een)i(used)f(so)f(far)g(with)h (deriv)o(ed)g(datat)o(yp)q(es)f(that)g(are)g(constructed)g(from)g (basic)h(datat)o(yp)q(es)f(using)75 158 y(the)j(constructors)e(describ) q(ed)j(in)g(this)f(section.)19 b(These)13 b(metho)q(ds)f(of)g (constructing)h(deriv)o(ed)h(datat)o(yp)q(es)75 214 y(can)h(b)q(e)h (applied)h(recursiv)o(ely)l(.)166 271 y(A)e Fx(general)j(datat)o(yp)q (e)e FC(is)g(an)f(opaque)h(ob)s(ject)e(that)h(sp)q(eci\014es)i(t)o(w)o (o)d(things:)143 366 y FA(\017)23 b FC(A)15 b(sequence)h(of)f(basic)h (datat)o(yp)q(es)143 461 y FA(\017)23 b FC(A)15 b(sequence)h(of)f(in)o (teger)g(\(b)o(yte\))g(displacemen)o(ts)166 557 y(The)22 b(displacemen)o(ts)g(are)g(not)f(required)h(to)f(b)q(e)h(p)q(ositiv)o (e,)i(distinct,)g(or)d(in)h(increasing)h(order;)75 613 y(therefore)17 b(the)g(order)g(of)f(items)i(need)g(not)f(coincide)i (with)e(their)h(order)f(in)g(store,)g(and)g(an)g(item)h(ma)o(y)75 670 y(app)q(ear)c(more)g(than)g(once.)20 b(W)l(e)14 b(call)h(suc)o(h)g (pair)f(of)g(sequences)h(\(or)e(sequence)j(of)d(pairs\))h(a)g Fx(t)o(yp)q(e)i(map)p FC(.)75 726 y(The)22 b(sequence)g(of)f(basic)h (datat)o(yp)q(es)f(\(displacemen)o(ts)h(ignored\))g(is)f(the)h Fx(t)o(yp)q(e)i(signature)e FC(of)f(the)75 782 y(datat)o(yp)q(e.)166 839 y(Let)189 935 y Fw(T)6 b(y)r(pemap)12 b FC(=)h FA(f)p FC(\()p Fw(ty)r(pe)562 942 y Fi(0)581 935 y Fw(;)8 b(disp)686 942 y Fi(0)705 935 y FC(\))p Fw(;)g(:::;)g FC(\()o Fw(ty)r(pe)905 942 y Fh(n)p Fj(\000)p Fi(1)971 935 y Fw(;)g(disp)1076 942 y Fh(n)p Fj(\000)p Fi(1)1144 935 y FC(\))p FA(g)p Fw(;)75 1030 y FC(b)q(e)16 b(suc)o(h)f(a)g(t)o(yp)q(e)h(map,)e(where)i Fw(ty)r(pe)709 1037 y Fh(i)738 1030 y FC(are)f(basic)h(t)o(yp)q(es,)f (and)g Fw(disp)1233 1037 y Fh(i)1262 1030 y FC(are)g(displacemen)o(ts.) 21 b(Let)189 1125 y Fw(T)6 b(y)r(pesig)13 b FC(=)g FA(f)p Fw(ty)r(pe)517 1132 y Fi(0)537 1125 y Fw(;)8 b(:::;)g(ty)r(pe)702 1132 y Fh(n)p Fj(\000)p Fi(1)768 1125 y FA(g)75 1220 y FC(b)q(e)14 b(the)f(asso)q(ciated)h(t)o(yp)q(e)f(signature.)19 b(This)14 b(t)o(yp)q(e)f(map,)g(together)g(with)g(a)g(base)g(address)h Fr(buf)p FC(,)f(sp)q(eci\014es)75 1277 y(a)19 b(comm)o(unication)i (bu\013er:)28 b(the)20 b(comm)o(unication)h(bu\013er)e(that)g(consists) h(of)f Fw(n)h FC(en)o(tries,)h(where)f(the)75 1333 y Fw(i)p FC(-th)e(en)o(try)g(is)g(at)g(address)g Fw(buf)f FC(+)12 b Fw(disp)777 1340 y Fh(i)809 1333 y FC(and)19 b(has)f(t)o(yp)q(e)g Fw(ty)r(pe)1174 1340 y Fh(i)1188 1333 y FC(.)29 b(A)19 b(message)e(assem)o(bled)i(from)f(suc)o(h)75 1390 y(comm)o(unication)e(bu\013er)f(will)i(consist)e(of)g Fw(n)h FC(v)m(alues,)g(of)e(the)i(t)o(yp)q(es)f(de\014ned)h(b)o(y)g Fw(T)6 b(y)r(pesig)r FC(.)166 1447 y(W)l(e)11 b(can)f(use)h(a)f(handle) i(to)e(a)g(general)h(datat)o(yp)q(e)f(as)h(an)f(argumen)o(t)g(in)h(a)f (send)i(or)e(receiv)o(e)h(op)q(eration,)75 1503 y(in)j(replacemen)o(t)f (of)f(a)h(basic)g(datat)o(yp)q(e)g(argumen)o(t.)18 b(The)13 b(op)q(eration)g Fv(MPI)p 1359 1503 14 2 v 16 w(SEND\(buf,)h(1,)e (datat)o(yp)q(e,...\))75 1559 y FC(will)j(use)e(the)h(send)g(bu\013er)f (de\014ned)h(b)o(y)g(the)f(base)g(address)h Fv(buf)g FC(and)f(the)g(general)h(datat)o(yp)q(e)f(asso)q(ciated)75 1616 y(with)f Fv(datat)o(yp)q(e)p FC(;)i(it)e(will)i(generate)d(a)h (message)f(with)h(the)g(t)o(yp)q(e)g(signature)g(determined)h(b)o(y)e (the)h Fv(datat)o(yp)q(e)75 1672 y FC(argumen)o(t.)28 b Fv(MPI)p 390 1672 V 16 w(RECV\(buf,)20 b(1,)e(datat)o(yp)q(e,...\))29 b FC(will)20 b(use)e(the)h(receiv)o(e)g(bu\013er)f(de\014ned)h(b)o(y)g (the)f(base)75 1729 y(address)d Fv(buf)h FC(and)g(the)f(general)h (datat)o(yp)q(e)e(asso)q(ciated)i(with)f Fv(datat)o(yp)q(e)p FC(.)166 1786 y(General)h(datat)o(yp)q(es)e(can)i(b)q(e)g(used)f(in)h (all)h(send)e(and)h(receiv)o(e)g(op)q(erations.)k(W)l(e)c(address)f (later)g(in)75 1842 y(section)h(3.13.5)d(the)i(case)h(where)f(the)g (second)h(argumen)o(t)e Fv(count)j FC(has)e(v)m(alue)h Fw(>)d FC(1.)166 1899 y(The)e(basic)g(datat)o(yp)q(es)f(presen)o(ted)i (in)f(section)g(3.2.1)f(are)g(particular)h(cases)g(of)f(a)h(general)g (datat)o(yp)q(e,)75 1955 y(and)19 b(are)f(prede\014ned.)32 b(Th)o(us,)19 b Fv(MPI)p 711 1955 V 16 w(INT)f FC(is)h(a)g (prede\014ned)h(handle)g(to)e(a)g(datat)o(yp)q(e)g(with)h(t)o(yp)q(e)g (map)75 2012 y FA(f)p FC(\()p Fo(int)o Fw(;)8 b FC(0\))p FA(g)p FC(,)16 b(with)h(one)h(en)o(try)f(of)g(t)o(yp)q(e)g Fo(int)g FC(and)g(displacemen)o(t)i(zero.)26 b(And)18 b(similarly)h(for)e(all)h(other)75 2068 y(basic)e(datat)o(yp)q(es.)166 2125 y(The)f Fx(exten)o(t)f FC(of)g(a)g(datat)o(yp)q(e)g(is)h (de\014ned)g(to)f(b)q(e)h(the)g(span)f(from)g(the)g(\014rst)g(b)o(yte)h (to)e(the)i(last)f(b)o(yte)75 2181 y(o)q(ccupied)19 b(b)o(y)d(en)o (tries)h(in)h(this)f(datat)o(yp)q(e,)f(rounded)h(up)g(to)f(satisfy)h (alignmen)o(t)g(requiremen)o(ts.)25 b(That)75 2238 y(is,)15 b(if)189 2333 y Fw(T)6 b(y)r(pemap)12 b FC(=)h FA(f)p FC(\()p Fw(ty)r(pe)562 2340 y Fi(0)581 2333 y Fw(;)8 b(disp)686 2340 y Fi(0)705 2333 y FC(\))p Fw(;)g(:::;)g FC(\()o Fw(ty)r(pe)905 2340 y Fh(n)p Fj(\000)p Fi(1)971 2333 y Fw(;)g(disp)1076 2340 y Fh(n)p Fj(\000)p Fi(1)1144 2333 y FC(\))p FA(g)p Fw(;)75 2428 y FC(then)283 2523 y Fw(l)q(b)p FC(\()p Fw(T)e(y)r(pemap)p FC(\))40 b(=)h(min)688 2552 y Fh(j)742 2523 y Fw(disp)826 2530 y Fh(j)844 2523 y Fw(;)272 2613 y(ub)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))40 b(=)h(max)693 2642 y Fh(j)743 2613 y FC(\()p Fw(disp)845 2620 y Fh(j)873 2613 y FC(+)10 b Fw(siz)r(eof)5 b FC(\()p Fw(ty)r(pe)1150 2620 y Fh(j)1169 2613 y FC(\)\))p Fw(;)21 b FC(and)189 2704 y Fw(extent)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))42 b(=)f Fw(ub)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))j FA(\000)h Fw(l)q(b)p FC(\()p Fw(T)c(y)r(pemap)p FC(\))j(+)h Fw(\017:)456 b FC(\(3.1\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 46 54 46 53 bop 75 -100 a FC(46)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y FC(If)h Fw(ty)r(pe)206 52 y Fh(i)237 45 y FC(requires)g(alignmen)o(t)g(to)f(a)h (b)o(yte)f(address)h(that)f(is)h(is)g(a)f(m)o(ultiple)j(of)d Fw(k)1474 52 y Fh(i)1488 45 y FC(,)g(then)h Fw(\017)g FC(is)g(the)g(least)75 102 y(nonnegativ)o(e)f(incremen)o(t)g(needed)g (to)f(round)g Fw(extent)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))16 b(to)e(the)h(next)h(m)o(ultiple)h(of)d(max)1794 109 y Fh(i)1816 102 y Fw(k)1840 109 y Fh(i)1854 102 y FC(.)166 162 y(Example:)k(Assume)10 b(that)g Fw(T)c(y)r(pe)13 b FC(=)g FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p FA(g)h FC(\(a)g Fo(double)h FC(at)g(displacemen)o(t)h(zero,)75 219 y(follo)o(w)o(ed)17 b(b)o(y)g(a)f Fo(char)g FC(at)g(displacemen)o(t)i(eigh)o(t\).)24 b(Assume,)17 b(furthermore,)f(that)g(doubles)i(ha)o(v)o(e)e(to)g(b)q(e) 75 275 y(strictly)f(aligned)g(at)f(addresses)h(that)e(are)h(m)o (ultiple)i(of)e(eigh)o(t.)20 b(Then,)15 b(the)f(exten)o(t)g(of)g(this)h (datat)o(yp)q(e)e(is)75 331 y(16)d(\(9)g(rounded)h(to)f(the)h(next)g(m) o(ultiple)h(of)f(8\).)17 b(A)11 b(datat)o(yp)q(e)f(that)g(consists)h (of)f(a)g(c)o(haracter)g(immediately)75 388 y(follo)o(w)o(ed)16 b(b)o(y)f(a)g(double)h(will)h(also)e(ha)o(v)o(e)g(an)g(exten)o(t)g(of)g (16.)75 533 y Fp(3.13.1)49 b(Datat)o(yp)q(e)17 b(constructo)o(rs)75 627 y Fv(Contiguous)47 b FC(The)16 b(simplest)i(datat)o(yp)q(e)e (constructor)f(is)i Fv(MPI)p 1171 627 14 2 v 16 w(TYPE)p 1304 627 V 17 w(CONTIGUOUS)h FC(whic)o(h)f(allo)o(ws)75 683 y(replication)g(of)e(a)g(datat)o(yp)q(e)f(in)o(to)h(con)o(tiguous)h (lo)q(cations.)75 791 y Fv(MPI)p 160 791 V 16 w(TYPE)p 293 791 V 17 w(CONTIGUOUS\(count,)h(oldt)o(yp)q(e,)f(newt)o(yp)q(e\)) 117 873 y Fq(IN)155 b Fv(count)482 b Fq(replication)14 b(coun)o(t)g(\(nonnegativ)o(e)g(in)o(teger\))117 956 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fq(old)13 b(datat)o(yp)q(e)i (\(handle\))117 1039 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fq(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1168 y Fo(int)23 b(MPI)p 245 1168 15 2 v 17 w(Type)p 358 1168 V 17 w(contiguous\(int)f (count,)h(MPI)p 972 1168 V 17 w(Datatype)f(oldtype,)393 1224 y(MPI)p 468 1224 V 17 w(Datatype)h(*newtype\))75 1315 y(MPI)p 150 1315 V 17 w(TYPE)p 263 1315 V 16 w(CONTIGUOUS\(COUNT,) f(OLDTYPE,)h(NEWTYPE,)f(IERROR\))170 1371 y(INTEGER)h(COUNT,)g (OLDTYPE,)g(NEWTYPE,)g(IERROR)166 1462 y Fv(newt)o(yp)q(e)c FC(is)f(the)f(datat)o(yp)q(e)g(obtained)h(b)o(y)f(concatenating)h Fv(count)h FC(copies)f(of)f Fv(oldt)o(yp)q(e)p FC(.)27 b(Concate-)75 1518 y(nation)15 b(is)h(de\014ned)h(using)f Fr(extent)e FC(as)h(the)h(size)g(of)e(the)i(concatenated)f(copies.)166 1579 y(Example:)26 b(let)18 b Fv(oldt)o(yp)q(e)h FC(ha)o(v)o(e)e(t)o (yp)q(e)h(map)g FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p FA(g)p Fw(;)16 b FC(with)i(exten)o(t)g(16,)f(and)h(let)75 1635 y Fv(count)c FC(=)f(3.)19 b(The)d(t)o(yp)q(e)f(map)g(of)g(the)g(datat)o(yp)q(e)g (returned)g(b)o(y)h Fv(newt)o(yp)q(e)h FC(is)189 1750 y FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FC(16\))p Fw(;)g FC(\()p Fo(char)l Fw(;)g FC(24\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FC(32\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(40\))p FA(g)p FC(;)75 1860 y(i.e.,)15 b(alternating)g Fo(double)g FC(and)g Fo(char)g FC(elemen)o(ts,)g(with)h (displacemen)o(ts)g(0)p Fw(;)8 b FC(8)p Fw(;)g FC(16)p Fw(;)g FC(24)o Fw(;)f FC(32)p Fw(;)g FC(40.)166 1921 y(In)16 b(general,)f(assume)g(that)g(the)g(t)o(yp)q(e)g(map)g(of)g Fv(oldt)o(yp)q(e)h FC(is)189 2035 y FA(f)p FC(\()p Fw(ty)r(pe)314 2042 y Fi(0)333 2035 y Fw(;)8 b(disp)438 2042 y Fi(0)457 2035 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 2042 y Fh(n)p Fj(\000)p Fi(1)723 2035 y Fw(;)h(disp)828 2042 y Fh(n)p Fj(\000)p Fi(1)895 2035 y FC(\))p FA(g)p Fw(;)75 2145 y FC(with)16 b(exten)o(t)e Fw(extent)p FC(.)22 b(Then)15 b Fv(newt)o(yp)q(e)i FC(has)e(a)g(t)o(yp)q(e)h(map)f(with)g Fo(count)9 b FA(\001)h Fo(n)15 b FC(en)o(tries)h(de\014ned)g(b)o(y:)189 2260 y FA(f)p FC(\()p Fw(ty)r(pe)314 2267 y Fi(0)333 2260 y Fw(;)8 b(disp)438 2267 y Fi(0)457 2260 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 2267 y Fh(n)p Fj(\000)p Fi(1)723 2260 y Fw(;)h(disp)828 2267 y Fh(n)p Fj(\000)p Fi(1)895 2260 y FC(\))p Fw(;)g FC(\()p Fw(ty)r(pe)1036 2267 y Fi(0)1055 2260 y Fw(;)g(disp)1160 2267 y Fi(0)1189 2260 y FC(+)i Fw(extent)p FC(\))p Fw(;)e(:::;)g FC(\()p Fw(ty)r(pe)1562 2267 y Fh(n)p Fj(\000)p Fi(1)1629 2260 y Fw(;)g(disp)1734 2267 y Fh(n)p Fj(\000)p Fi(1)1811 2260 y FC(+)j Fw(extent)p FC(\))p Fw(;)189 2391 y(:::;)d FC(\()p Fw(ty)r(pe)351 2398 y Fi(0)368 2391 y Fw(;)g(disp)473 2398 y Fi(0)502 2391 y FC(+)i Fw(extent)i FA(\001)e FC(\()p Fo(count)f FA(\000)h FC(1\)\))p Fw(;)e(:::;)g FC(\()o Fw(ty)r(pe)1142 2398 y Fh(n)p Fj(\000)p Fi(1)1208 2391 y Fw(;)g(disp)1313 2398 y Fh(n)p Fj(\000)p Fi(1)1391 2391 y FC(+)i Fw(extent)i FA(\001)d FC(\()p Fo(count)g FA(\000)i FC(1\)\))p FA(g)p Fw(:)75 2534 y Fv(V)o(ecto)o(r)45 b FC(The)16 b(function)g Fv(MPI)p 600 2534 14 2 v 16 w(TYPE)p 733 2534 V 17 w(VECTOR)g FC(is)g(a)f(more)g(general)h (constructor)e(that)h(allo)o(ws)g(repli-)75 2591 y(cation)20 b(of)g(a)g(datat)o(yp)q(e)f(in)o(to)h(lo)q(cations)h(that)e(consist)i (of)e(equally)j(spaced)f(blo)q(c)o(ks.)35 b(Eac)o(h)20 b(blo)q(c)o(k)g(is)75 2647 y(obtained)g(b)o(y)f(concatenating)h(the)f (same)g(n)o(um)o(b)q(er)h(of)f(copies)h(of)f(the)g(old)h(datat)o(yp)q (e.)32 b(The)19 b(spacing)75 2704 y(b)q(et)o(w)o(een)d(blo)q(c)o(ks)f (is)h(a)f(m)o(ultiple)i(of)e(the)g(exten)o(t)g(of)g(the)g(old)h(datat)o (yp)q(e.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 47 55 47 54 bop 75 -100 a Fs(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(47)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(VECTOR\()16 b(count,)g(blo)q(cklength,)g(stride,)g(oldt)o (yp)q(e,)g(newt)o(yp)q(e\))117 122 y Fq(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o (teger\))117 194 y(IN)155 b Fv(blo)q(cklength)371 b Fq(n)o(um)o(b)q(er) 16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 250 y(ger\))117 322 y(IN)155 b Fv(stride)484 b Fq(n)o(um)o(b)q(er)13 b(of)f(elemen)o(ts)h(b)q(et)o(w)o(een)h(start)g (of)f(eac)o(h)g(blo)q(c)o(k)g(\(in)o(te-)905 378 y(ger\))117 450 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fq(old)13 b(datat)o(yp)q(e)i (\(handle\))117 522 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fq(new)15 b(datat)o(yp)q(e)f(\(handle\))75 646 y Fo(int)23 b(MPI)p 245 646 15 2 v 17 w(Type)p 358 646 V 17 w(vector\(int)f(count,) h(int)h(blocklength,)e(int)i(stride,)393 702 y(MPI)p 468 702 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 702 V 17 w(Datatype)h(*newtype\))75 789 y(MPI)p 150 789 V 17 w(TYPE)p 263 789 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)g (OLDTYPE,)f(NEWTYPE,)h(IERROR\))170 845 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)166 932 y FC(Example:)35 b(Assume,)24 b(again,)f(that)f Fv(oldt)o(yp)q(e)h FC(has)f(t)o(yp)q(e)h(map)f FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p FA(g)p Fw(;)21 b FC(with)75 988 y(exten)o(t)13 b(16.)18 b(A)13 b(call)h(to)e Fv(MPI)p 555 988 14 2 v 16 w(TYPE)p 688 988 V 17 w(VECTOR\()h(2,)g(3,)g(4,)f(oldt)o(yp)q(e,)i(newt)o(yp)q (e\))g FC(will)h(create)e(the)f(datat)o(yp)q(e)75 1045 y(with)k(t)o(yp)q(e)f(map)189 1121 y FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FC(16\))p Fw(;)g FC(\()p Fo(char)l Fw(;)g FC(24\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FC(32\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(40\))p Fw(;)189 1202 y FC(\()p Fo(double)n Fw(;)g FC(64\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(72\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FC(80\))p Fw(;)g FC(\()p Fo(ch)o(ar)m Fw(;)g FC(88\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FC(96\))p Fw(;)g FC(\()p Fo(cha)o(r)m Fw(;)g FC(104\))p FA(g)j FC(:)75 1284 y(t)o(w)o(o)j(blo)q(c)o(ks)i(with)f(three)h(copies)g(eac)o (h)f(of)g(the)g(old)h(t)o(yp)q(e,)f(starting)f(4*16)g(apart.)166 1340 y(A)h(call)h(to)f Fv(MPI)p 439 1340 V 16 w(TYPE)p 572 1340 V 17 w(VECTOR\(3,)g(1,)g(-2,)f(oldt)o(yp)q(e,)i(newt)o(yp)q (e\))h FC(will)g(create)e(the)g(datat)o(yp)q(e)189 1416 y FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FA(\000)p FC(32\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FA(\000)p FC(24\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FA(\000)p FC(64\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FA(\000)p FC(56\))p FA(g)p Fw(:)166 1492 y FC(In)16 b(general,)f(assume)g(that)g Fv(oldt)o(yp)q(e)h FC(has)f(t)o(yp)q(e)g(map)189 1568 y FA(f)p FC(\()p Fw(ty)r(pe)314 1575 y Fi(0)333 1568 y Fw(;)8 b(disp)438 1575 y Fi(0)457 1568 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 1575 y Fh(n)p Fj(\000)p Fi(1)723 1568 y Fw(;)h(disp)828 1575 y Fh(n)p Fj(\000)p Fi(1)895 1568 y FC(\))p FA(g)p Fw(;)75 1644 y FC(with)k(exten)o(t)e Fw(extent)p FC(.)20 b(The)12 b(newly)g(created)f(datat)o(yp)q(e)g(has)g (a)g(t)o(yp)q(e)g(map)g(with)g Fo(count)r FA(\001)r Fo(blocklength)p FA(\001)r Fw(n)75 1701 y FC(en)o(tries:)189 1777 y FA(f)p FC(\()p Fw(ty)r(pe)314 1784 y Fi(0)333 1777 y Fw(;)d(disp)438 1784 y Fi(0)457 1777 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 1784 y Fh(n)p Fj(\000)p Fi(1)723 1777 y Fw(;)h(disp)828 1784 y Fh(n)p Fj(\000)p Fi(1)895 1777 y FC(\))p Fw(;)189 1858 y FC(\()p Fw(ty)r(pe)291 1865 y Fi(0)310 1858 y Fw(;)g(disp)415 1865 y Fi(0)444 1858 y FC(+)i Fw(extent)p FC(\))p Fw(;)e(:::;)g FC(\()p Fw(ty)r(pe)817 1865 y Fh(n)p Fj(\000)p Fi(1)884 1858 y Fw(;)g(disp)989 1865 y Fh(n)p Fj(\000)p Fi(1)1067 1858 y FC(+)i Fw(extent)p FC(\))p Fw(;)e(:::;)189 1940 y FC(\()p Fw(ty)r(pe)291 1947 y Fi(0)310 1940 y Fw(;)g(disp)415 1947 y Fi(0)444 1940 y FC(+)i(\()p Fo(blocklength)f FA(\000)h FC(1\))g FA(\001)f Fw(extent)p FC(\))p Fw(;)f(:::;)g FC(\()p Fw(ty)r(pe)1226 1947 y Fh(n)p Fj(\000)p Fi(1)1293 1940 y Fw(;)g(disp)1398 1947 y Fh(n)p Fj(\000)p Fi(1)1476 1940 y FC(+)i(\()p Fo(blocklength)e FA(\000)j FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)189 2021 y FC(\()p Fw(ty)r(pe)291 2028 y Fi(0)310 2021 y Fw(;)e(disp)415 2028 y Fi(0)444 2021 y FC(+)i Fo(stride)g FA(\001)f Fw(extent)p FC(\))p Fw(;)f(:::;)g FC(\()p Fw(ty)r(pe)993 2028 y Fh(n)p Fj(\000)p Fi(1)1060 2021 y Fw(;)g(disp)1165 2028 y Fh(n)p Fj(\000)p Fi(1)1243 2021 y FC(+)i Fo(stride)f FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2102 y FC(\()p Fw(ty)r(pe)291 2109 y Fi(0)310 2102 y Fw(;)g(disp)415 2109 y Fi(0)444 2102 y FC(+)i(\()p Fo(stride)f FC(+)i Fo(blocklength)d FA(\000)j FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2184 y FC(\()p Fw(ty)r(pe)291 2191 y Fh(n)p Fj(\000)p Fi(1)359 2184 y Fw(;)g(disp)464 2191 y Fh(n)p Fj(\000)p Fi(1)542 2184 y FC(+)i(\()p Fo(stride)f FC(+)h Fo(blocklength)f FA(\000)i FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(::::;)189 2265 y FC(\()p Fw(ty)r(pe)291 2272 y Fi(0)310 2265 y Fw(;)g(disp)415 2272 y Fi(0)444 2265 y FC(+)i Fo(stride)g FA(\001)f FC(\()p Fo(count)g FA(\000)i FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2346 y FC(\()p Fw(ty)r(pe)291 2353 y Fh(n)p Fj(\000)p Fi(1)359 2346 y Fw(;)g(disp)464 2353 y Fh(n)p Fj(\000)p Fi(1)542 2346 y FC(+)i Fo(stride)f FA(\001)h FC(\()p Fo(count)f FA(\000)i FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2428 y FC(\()p Fw(ty)r(pe)291 2435 y Fi(0)310 2428 y Fw(;)g(disp)415 2435 y Fi(0)444 2428 y FC(+)i(\()p Fo(stride)f FA(\001)h FC(\()p Fo(count)f FA(\000)i FC(1\))e(+)i Fo(blocklength)d FA(\000)j FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2509 y FC(\()p Fw(ty)r(pe)291 2516 y Fh(n)p Fj(\000)p Fi(1)359 2509 y Fw(;)g(disp)464 2516 y Fh(n)p Fj(\000)p Fi(1)542 2509 y FC(+)i(\()p Fo(stride)f FA(\001)h FC(\()p Fo(count)f FA(\000)h FC(1\))g(+)g Fo(blocklength)f FA(\000)h FC(1\))g FA(\001)g Fw(extent)p FC(\))p FA(g)166 2591 y FC(A)16 b(call)i(to)e Fv(MPI)p 443 2591 V 16 w(TYPE)p 576 2591 V 16 w(CONTIGUOUS\(count,)k(oldt)o(yp)q(e,)d(newt)o(yp)q(e\))h FC(is)f(equiv)m(alen)o(t)h(to)e(a)g(call)i(to)75 2647 y Fv(MPI)p 160 2647 V 16 w(TYPE)p 293 2647 V 17 w(VECTOR\(count,)g(1,)f (1,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p FC(,)h(or)e(to)g(a)h(call)g (to)f Fv(MPI)p 1495 2647 V 16 w(TYPE)p 1628 2647 V 17 w(VECTOR\(1,)75 2704 y(count,)g(n,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p FC(,)h Fo(n)e FC(arbitrary)l(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 48 56 48 55 bop 75 -100 a FC(48)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fv(Hvecto)o(r)45 b FC(The)19 b(function)h Fv(MPI)p 631 45 14 2 v 16 w(TYPE)p 764 45 V 17 w(HVECTOR)g FC(is)f(iden)o(tical)i(to)d Fv(MPI)p 1389 45 V 16 w(TYPE)p 1522 45 V 17 w(VECTOR)p FC(,)h(except)75 102 y(that)h Fv(stride)h FC(is)g(giv)o(en)g(in)g(b)o(ytes,)g(rather)f (then)g(in)i(elemen)o(ts.)36 b(The)20 b(use)h(for)f(b)q(oth)g(t)o(yp)q (es)h(of)f(v)o(ector)75 158 y(constructors)14 b(is)i(illustrated)h(in)f (section)f(3.13.6.)j(\()p Fo(H)d FC(is)h(used)g(for)e (\\heterogeneous"\).)75 266 y Fv(MPI)p 160 266 V 16 w(TYPE)p 293 266 V 17 w(HVECTOR\()i(count,)g(blo)q(cklength,)h(stride,)e(oldt)o (yp)q(e,)h(newt)o(yp)q(e\))117 348 y Fq(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o (teger\))117 432 y(IN)155 b Fv(blo)q(cklength)371 b Fq(n)o(um)o(b)q(er) 16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 488 y(ger\))117 572 y(IN)155 b Fv(stride)484 b Fq(n)o(um)o(b)q(er)13 b(of)g(b)o(ytes)h(b)q(et)o(w)o(een)h(start)f (of)f(eac)o(h)h(blo)q(c)o(k)f(\(in)o(teger\))905 628 y FC(funcarg)p Fq(IN)p FC(oldt)o(yp)q(eold)k(datat)o(yp)q(e)d (\(handle\))117 712 y Fq(OUT)108 b Fv(newt)o(yp)q(e)433 b Fq(new)15 b(datat)o(yp)q(e)f(\(handle\))75 841 y Fo(int)23 b(MPI)p 245 841 15 2 v 17 w(Type)p 358 841 V 17 w(hvector\(int)f (count,)h(int)h(blocklength,)e(MPI)p 1306 841 V 17 w(Aint)h(stride,)393 897 y(MPI)p 468 897 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 897 V 17 w(Datatype)h(*newtype\))75 988 y(MPI)p 150 988 V 17 w(TYPE)p 263 988 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)f(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1045 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)166 1136 y FC(Assume)15 b(that)g Fv(oldt)o(yp)q(e)h FC(has)f(t)o(yp)q(e)g (map)189 1251 y FA(f)p FC(\()p Fw(ty)r(pe)314 1258 y Fi(0)333 1251 y Fw(;)8 b(disp)438 1258 y Fi(0)457 1251 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 1258 y Fh(n)p Fj(\000)p Fi(1)723 1251 y Fw(;)h(disp)828 1258 y Fh(n)p Fj(\000)p Fi(1)895 1251 y FC(\))p FA(g)p Fw(;)75 1363 y FC(with)k(exten)o(t)e Fw(extent)p FC(.)20 b(The)12 b(newly)g(created)f(datat)o(yp)q(e)g(has)g(a)g(t)o(yp)q(e)g(map)g(with) g Fo(count)r FA(\001)r Fo(blocklength)p FA(\001)r Fw(n)75 1419 y FC(en)o(tries:)189 1535 y FA(f)p FC(\()p Fw(ty)r(pe)314 1542 y Fi(0)333 1535 y Fw(;)d(disp)438 1542 y Fi(0)457 1535 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 1542 y Fh(n)p Fj(\000)p Fi(1)723 1535 y Fw(;)h(disp)828 1542 y Fh(n)p Fj(\000)p Fi(1)895 1535 y FC(\))p Fw(;)189 1667 y FC(\()p Fw(ty)r(pe)291 1674 y Fi(0)310 1667 y Fw(;)g(disp)415 1674 y Fi(0)444 1667 y FC(+)i Fw(extent)p FC(\))p Fw(;)e(:::;)g FC(\()p Fw(ty)r(pe)817 1674 y Fh(n)p Fj(\000)p Fi(1)884 1667 y Fw(;)g(disp)989 1674 y Fh(n)p Fj(\000)p Fi(1)1067 1667 y FC(+)i Fw(extent)p FC(\))p Fw(;)e(:::;)189 1795 y FC(\()p Fw(ty)r(pe)291 1802 y Fi(0)310 1795 y Fw(;)g(disp)415 1802 y Fi(0)444 1795 y FC(+)i(\()p Fo(blocklength)f FA(\000)h FC(1\))g FA(\001)f Fw(extent)p FC(\))p Fw(;)f(:::;)g FC(\()p Fw(ty)r(pe)1226 1802 y Fh(n)p Fj(\000)p Fi(1)1293 1795 y Fw(;)g(disp)1398 1802 y Fh(n)p Fj(\000)p Fi(1)1476 1795 y FC(+)i(\()p Fo(blocklength)e FA(\000)j FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)189 1923 y FC(\()p Fw(ty)r(pe)291 1930 y Fi(0)310 1923 y Fw(;)e(disp)415 1930 y Fi(0)444 1923 y FC(+)i Fo(stride)p FC(\))p Fw(;)e(:::;)g FC(\()o Fw(ty)r(pe)833 1930 y Fh(n)p Fj(\000)p Fi(1)899 1923 y Fw(;)g(disp)1004 1930 y Fh(n)p Fj(\000)p Fi(1)1081 1923 y FC(+)j Fo(stride)o FC(\))p Fw(;)d(:::;)189 2052 y FC(\()p Fw(ty)r(pe)291 2059 y Fi(0)310 2052 y Fw(;)g(disp)415 2059 y Fi(0)444 2052 y FC(+)i Fo(stride)g FC(+)g(\()p Fo(blocklength)e FA(\000)j FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2180 y FC(\()p Fw(ty)r(pe)291 2187 y Fh(n)p Fj(\000)p Fi(1)359 2180 y Fw(;)g(disp)464 2187 y Fh(n)p Fj(\000)p Fi(1)542 2180 y FC(+)i Fo(stride)f FC(+)i(\()p Fo(blocklength)d FA(\000)j FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(::::;)189 2308 y FC(\()p Fw(ty)r(pe)291 2315 y Fi(0)310 2308 y Fw(;)g(disp)415 2315 y Fi(0)444 2308 y FC(+)i Fo(stride)g FA(\001)f FC(\()p Fo(count)g FA(\000)i FC(1\)\))p Fw(;)d(:::;)189 2436 y FC(\()p Fw(ty)r(pe)291 2443 y Fh(n)p Fj(\000)p Fi(1)359 2436 y Fw(;)g(disp)464 2443 y Fh(n)p Fj(\000)p Fi(1)542 2436 y FC(+)i Fo(stride)f FA(\001)h FC(\()p Fo(count)f FA(\000)i FC(1\)\))p Fw(;)d(:::;)189 2564 y FC(\()p Fw(ty)r(pe)291 2571 y Fi(0)310 2564 y Fw(;)g(disp)415 2571 y Fi(0)444 2564 y FC(+)i Fo(stride)g FA(\001)f FC(\()p Fo(count)g FA(\000)i FC(1\))e(+)i(\()p Fo(blocklength)d FA(\000)j FC(1\))e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2692 y FC(\()p Fw(ty)r(pe)291 2699 y Fh(n)p Fj(\000)p Fi(1)359 2692 y Fw(;)g(disp)464 2699 y Fh(n)p Fj(\000)p Fi(1)542 2692 y FC(+)i Fo(stride)f FA(\001)h FC(\()p Fo(count)f FA(\000)i FC(1\))e(+)i(\()p Fo(blocklength)d FA(\000)i FC(1\))g FA(\001)g Fw(extent)p FC(\))p FA(g)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 49 57 49 56 bop 75 -100 a Fs(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(49)75 45 y Fv(Indexed)47 b FC(The)19 b(function)h Fv(MPI)p 629 45 14 2 v 16 w(TYPE)p 762 45 V 17 w(INDEXED)f FC(allo)o(ws)h(replication)h(of)d(an)i(old)f(datat)o(yp)q(e)g(in)o(to)g (a)75 102 y(sequence)c(of)e(con)o(tiguous)h(blo)q(c)o(ks,)g(where)g (eac)o(h)g(blo)q(c)o(k)h(can)f(con)o(tain)g(di\013eren)o(t)g(n)o(um)o (b)q(er)g(of)f(copies)i(and)75 158 y(ha)o(v)o(e)f(a)g(di\013eren)o(t)g (displacemen)o(t.)21 b(All)16 b(blo)q(c)o(k)f(displacemen)o(ts)g(are)f (m)o(ultiples)i(of)e(the)h(old)f(t)o(yp)q(e)h(exten)o(t.)75 262 y Fv(MPI)p 160 262 V 16 w(TYPE)p 293 262 V 17 w(INDEXED\()d(count,) i(a)o(rra)o(y)p 762 262 V 14 w(of)p 813 262 V 16 w(blo)q(cklengths,)h (a)o(rra)o(y)p 1185 262 V 15 w(of)p 1237 262 V 16 w(displacements,)g (oldt)o(yp)q(e,)f(newt)o(yp)q(e\))117 396 y Fq(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f (n)o(um)o(b)q(er)h(of)f(en)o(tries)i(in)905 452 y Fl(a)o(rra)o(y)p 992 452 13 2 v 15 w(of)p 1041 452 V 15 w(displacements)19 b Fq(and)f Fl(a)o(rra)o(y)p 1483 452 V 15 w(of)p 1532 452 V 14 w(blo)q(cklengths)i Fq(\(in)o(te-)905 509 y(ger\))117 584 y(IN)155 b Fv(a)o(rra)o(y)p 416 584 14 2 v 15 w(of)p 468 584 V 16 w(blo)q(cklengths)191 b Fq(n)o(um)o(b)q(er)18 b(of)f(elemen)o(ts)h(p)q(er)i(blo)q(c)o(k)d(\(arra)o(y)i(of)e (nonnegativ)o(e)905 641 y(in)o(tegers\))117 716 y(IN)155 b Fv(a)o(rra)o(y)p 416 716 V 15 w(of)p 468 716 V 16 w(displacements)164 b Fq(displacemen)o(t)19 b(for)h(eac)o(h)g(blo)q(c)o(k,)g(in)g(m)o (ultiples)e(of)h Fl(oldt)o(yp)q(e)905 773 y Fq(exten)o(t)c(\(arra)o(y)f (of)f(in)o(teger\))117 848 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fq(old)13 b(datat)o(yp)q(e)i(\(handle\))117 924 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fq(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1049 y Fo(int)23 b(MPI)p 245 1049 15 2 v 17 w(Type)p 358 1049 V 17 w(indexed\(int)f(count,)h(int)h(*array)p 1068 1049 V 16 w(of)p 1132 1049 V 17 w(blocklengths,)393 1105 y(int)g(*array)p 636 1105 V 16 w(of)p 700 1105 V 17 w(displacements,)e(MPI)p 1147 1105 V 17 w(Datatype)g(oldtype,)393 1162 y(MPI)p 468 1162 V 17 w(Datatype)h(*newtype\))75 1248 y(MPI)p 150 1248 V 17 w(TYPE)p 263 1248 V 16 w(INDEXED\(COUNT,)f (ARRAY)p 757 1248 V 17 w(OF)p 822 1248 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 1248 V 16 w(OF)p 1357 1248 V 17 w(DISPLACEMENTS,)393 1305 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1361 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1361 V 17 w(OF)p 716 1361 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1361 V 17 w(OF)p 1323 1361 V 17 w(DISPLACEMENTS\(*\),)170 1418 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)166 1504 y FC(Example:)29 b(let)20 b Fv(oldt)o(yp)q(e)h FC(ha)o(v)o(e)e(t)o(yp)q(e)g(map)h FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p FA(g)p Fw(;)18 b FC(with)i(exten)o(t)f(16.)32 b(Let)20 b Fo(D)75 1561 y(=)k(\(4,)f(0\))17 b FC(and)g(let)h Fo(B)24 b(=)g(\(3,)f(1\))p FC(.)j(A)17 b(call)h(to)f Fv(MPI)p 998 1561 14 2 v 16 w(TYPE)p 1131 1561 V 17 w(INDEXED\(2,)f(B,) h(D,)g(oldt)o(yp)q(e,)i(newt)o(yp)q(e\))75 1617 y FC(returns)c(a)g (data)o(yp)q(e)g(with)g(t)o(yp)q(e)h(map)189 1712 y FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(64\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(72\))p Fw(;)g FC(\()p Fo(doubl)o(e)m Fw(;)g FC(80\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(88\))p Fw(;)g FC(\()p Fo(doubl)o(e)m Fw(;)g FC(96\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(104\))189 1820 y(\()p Fo(double)n Fw(;)g FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p FA(g)j FC(:)75 1927 y(three)k(copies)h(of)e(the)i(old)f(t)o(yp)q(e)g (starting)f(at)h(displacemen)o(t)h(64,)e(and)h(one)h(cop)o(y)e (starting)h(at)f(displace-)75 1984 y(men)o(t)h(0.)166 2040 y(In)h(general,)f(assume)g(that)g Fv(oldt)o(yp)q(e)h FC(has)f(t)o(yp)q(e)g(map)189 2135 y FA(f)p FC(\()p Fw(ty)r(pe)314 2142 y Fi(0)333 2135 y Fw(;)8 b(disp)438 2142 y Fi(0)457 2135 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 2142 y Fh(n)p Fj(\000)p Fi(1)723 2135 y Fw(;)h(disp)828 2142 y Fh(n)p Fj(\000)p Fi(1)895 2135 y FC(\))p FA(g)p Fw(;)75 2230 y FC(with)35 b(exten)o(t)g Fr(extent)p FC(.)79 b(Let)36 b Fv(B)f FC(b)q(e)h(the)f Fv(a)o(rra)o(y)p 1013 2230 V 15 w(of)p 1065 2230 V 16 w(blo)q(cklength)i FC(argumen)o(t)d(and)h Fv(D)g FC(b)q(e)h(the)75 2286 y Fv(a)o(rra)o(y)p 173 2286 V 15 w(of)p 225 2286 V 16 w(displacements)17 b FC(argumen)o(t.)i (The)c(newly)h(created)f(datat)o(yp)q(e)g(has)g Fw(n)c FA(\001)1450 2254 y Fg(P)1494 2267 y Ff(count)n Fj(\000)p Fi(1)1494 2299 y Fh(i)p Fi(=0)1647 2286 y Fv(B)p FC([)p Fv(i)p FC(])j(en)o(tries:)189 2381 y FA(f)p FC(\()p Fw(ty)r(pe)314 2388 y Fi(0)333 2381 y Fw(;)8 b(disp)438 2388 y Fi(0)467 2381 y FC(+)i Fv(D)p FC([)p Fv(0)p FC(])f FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)g FC(\()p Fw(ty)r(pe)954 2388 y Fh(n)p Fj(\000)p Fi(1)1021 2381 y Fw(;)g(disp)1126 2388 y Fh(n)p Fj(\000)p Fi(1)1203 2381 y FC(+)j Fv(D)p FC([)p Fv(0)p FC(])d FA(\001)i Fw(extent)p FC(\))p Fw(;)e(:::;)189 2489 y FC(\()p Fw(ty)r(pe)291 2496 y Fi(0)310 2489 y Fw(;)g(disp)415 2496 y Fi(0)444 2489 y FC(+)i(\()p Fv(D)p FC([)p Fv(0)p FC(])f(+)h Fv(B)p FC([)p Fv(0)p FC(])g FA(\000)g FC(1\))g FA(\001)f Fw(extent)p FC(\))p Fw(;)f(:::;)g FC(\()p Fw(ty)r(pe)1178 2496 y Fh(n)p Fj(\000)p Fi(1)1245 2489 y Fw(;)g(disp)1350 2496 y Fh(n)p Fj(\000)p Fi(1)1428 2489 y FC(+)i(\()p Fv(D)p FC([)p Fv(0)p FC(])f(+)h Fv(B)p FC([)p Fv(0)p FC(])g FA(\000)g FC(1\))g FA(\001)f Fw(extent)p FC(\))p Fw(;)f(:::;)189 2596 y FC(\()p Fw(ty)r(pe)291 2603 y Fi(0)310 2596 y Fw(;)g(disp)415 2603 y Fi(0)444 2596 y FC(+)i Fv(D)p FC([)p Fv(count)h FA(\000)g Fv(1)p FC(])e FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)g FC(\()p Fw(ty)r(pe)1093 2603 y Fh(n)p Fj(\000)p Fi(1)1160 2596 y Fw(;)g(disp)1265 2603 y Fh(n)p Fj(\000)p Fi(1)1342 2596 y FC(+)j Fv(D)p FC([)p Fv(count)g FA(\000)f Fv(1)p FC(])f FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2704 y FC(\()p Fw(ty)r(pe)291 2711 y Fi(0)310 2704 y Fw(;)g(disp)415 2711 y Fi(0)444 2704 y FC(+)i(\()p Fv(D)p FC([)p Fv(count)h FA(\000)f Fv(1)p FC(])g(+)g Fv(B)p FC([)p Fv(count)i FA(\000)e Fv(1)p FC(])g FA(\000)g FC(1\))f FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 50 58 50 57 bop 75 -100 a FC(50)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)189 45 y FC(\()p Fw(ty)r(pe)291 52 y Fh(n)p Fj(\000)p Fi(1)359 45 y Fw(;)8 b(disp)464 52 y Fh(n)p Fj(\000)p Fi(1)542 45 y FC(+)i(\()p Fv(D)p FC([)p Fv(count)h FA(\000)f Fv(1)p FC(])g(+)g Fv(B)p FC([)p Fv(count)h FA(\000)g Fv(1)p FC(])e FA(\000)i FC(1\))e FA(\001)h Fw(extent)p FC(\))p FA(g)p Fw(:)166 138 y FC(A)j(call)g(to)f Fv(MPI)p 431 138 14 2 v 16 w(TYPE)p 564 138 V 17 w(VECTOR\(count,)j(blo)q(cklength,)g(stride,)e(oldt)o(yp)q (e,)h(newt)o(yp)q(e\))g FC(is)f(equiv)m(alen)o(t)75 194 y(to)i(a)f(call)j(to)d Fv(MPI)p 392 194 V 16 w(TYPE)p 525 194 V 17 w(INDEX\(count,)i(B,)f(D,)f(oldt)o(yp)q(e,)i(newt)o(yp)q (e\))h FC(where)189 278 y Fv(D)p FC([)p Fv(j)p FC(])11 b(=)i Fw(j)g FA(\001)d Fv(stride)16 b Fw(;)8 b(j)14 b FC(=)f(0)p Fw(;)8 b(:::;)g Fo(cou)o(nt)f FA(\000)k FC(1)p Fw(;)75 362 y FC(and)189 446 y Fv(B)p FC([)p Fv(j)p FC(])h(=)h Fv(blo)q(cklength)k Fw(;)8 b(j)14 b FC(=)f(0)p Fw(;)8 b(:::;)g Fo(count)f FA(\000)j FC(1)p Fw(:)75 565 y Fv(Hindexed)47 b FC(The)11 b(function)h Fv(MPI)p 643 565 V 15 w(TYPE)p 775 565 V 17 w(HINDEXED)f FC(is)g(iden)o(tical)i(to)d Fv(MPI)p 1383 565 V 15 w(TYPE)p 1515 565 V 17 w(INDEXED)p FC(,)g(except)75 621 y(that)19 b(blo)q(c)o(k)i(displacemen)o(ts)h(in)41 b Fv(a)o(rra)o(y)p 774 621 V 14 w(of)p 825 621 V 16 w(displacements)22 b FC(are)e(sp)q(eci\014ed)i(in)f(b)o(ytes,)g(rather)e(than)h(in)75 678 y(m)o(ultiples)d(of)e(the)g Fv(oldt)o(yp)q(e)h FC(exten)o(t.)75 781 y Fv(MPI)p 160 781 V 16 w(TYPE)p 293 781 V 17 w(HINDEXED\()21 b(count,)k(a)o(rra)o(y)p 814 781 V 14 w(of)p 865 781 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1248 781 V 15 w(of)p 1300 781 V 16 w(displacements,)f(oldt)o(yp)q(e,)f(new-)75 838 y(t)o(yp)q(e\))117 915 y Fq(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f(n)o(um)o(b)q (er)h(of)f(en)o(tries)i(in)905 971 y Fl(a)o(rra)o(y)p 992 971 13 2 v 15 w(of)p 1041 971 V 15 w(displacements)19 b Fq(and)f Fl(a)o(rra)o(y)p 1483 971 V 15 w(of)p 1532 971 V 14 w(blo)q(cklengths)i Fq(\(in)o(te-)905 1028 y(ger\))117 1101 y(IN)155 b Fv(a)o(rra)o(y)p 416 1101 14 2 v 15 w(of)p 468 1101 V 16 w(blo)q(cklengths)191 b Fq(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)h(in)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)f(of)g (nonnega-)905 1157 y(tiv)o(e)f(in)o(tegers\))117 1231 y(IN)155 b Fv(a)o(rra)o(y)p 416 1231 V 15 w(of)p 468 1231 V 16 w(displacements)164 b Fq(b)o(yte)14 b(displacemen)o(t)g(of)f (eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1304 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fq(old)13 b(datat)o(yp)q(e)i (\(handle\))117 1377 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fq(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1501 y Fo(int)23 b(MPI)p 245 1501 15 2 v 17 w(Type)p 358 1501 V 17 w(hindexed\(int)f (count,)h(int)h(*array)p 1092 1501 V 16 w(of)p 1156 1501 V 17 w(blocklengths,)393 1558 y(MPI)p 468 1558 V 17 w(Aint)f(*array)p 748 1558 V 17 w(of)p 813 1558 V 17 w(displacements,)e(MPI)p 1259 1558 V 17 w(Datatype)i(oldtype,)393 1614 y(MPI)p 468 1614 V 17 w(Datatype)g(*newtype\))75 1701 y(MPI)p 150 1701 V 17 w(TYPE)p 263 1701 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 1701 V 17 w(OF)p 846 1701 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 1701 V 16 w(OF)p 1381 1701 V 17 w(DISPLACEMENTS,)393 1757 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1814 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1814 V 17 w(OF)p 716 1814 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1814 V 17 w(OF)p 1323 1814 V 17 w(DISPLACEMENTS\(*\),)170 1870 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)166 1957 y FC(Assume)15 b(that)g Fv(oldt)o(yp)q(e)h FC(has)f(t)o(yp)q(e)g(map)189 2041 y FA(f)p FC(\()p Fw(ty)r(pe)314 2048 y Fi(0)333 2041 y Fw(;)8 b(disp)438 2048 y Fi(0)457 2041 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 2048 y Fh(n)p Fj(\000)p Fi(1)723 2041 y Fw(;)h(disp)828 2048 y Fh(n)p Fj(\000)p Fi(1)895 2041 y FC(\))p FA(g)p Fw(;)75 2125 y FC(with)35 b(exten)o(t)g Fr(extent)p FC(.)79 b(Let)36 b Fv(B)f FC(b)q(e)h(the)f Fv(a)o(rra)o(y)p 1013 2125 14 2 v 15 w(of)p 1065 2125 V 16 w(blo)q(cklength)i FC(argumen)o(t)d(and)h Fv(D)g FC(b)q(e)h(the)75 2181 y Fv(a)o(rra)o(y)p 173 2181 V 15 w(of)p 225 2181 V 16 w(displacements)23 b FC(argumen)o(t.)39 b(The)22 b(newly)h(created)f(datat)o(yp)q(e)f(has)h(a)f(t)o(yp)q(e)h (map)f(with)i Fw(n)14 b FA(\001)75 2206 y Fg(P)119 2219 y Ff(count)n Fj(\000)p Fi(1)119 2251 y Fh(i)p Fi(=0)272 2238 y Fv(B)p FC([)p Fv(i)p FC(])g(en)o(tries:)189 2322 y FA(f)p FC(\()p Fw(ty)r(pe)314 2329 y Fi(0)333 2322 y Fw(;)8 b(disp)438 2329 y Fi(0)467 2322 y FC(+)i Fv(D)p FC([)p Fv(0)p FC(])o(\))p Fw(;)e(:::;)g FC(\()p Fw(ty)r(pe)794 2329 y Fh(n)p Fj(\000)p Fi(1)859 2322 y Fw(;)g(disp)964 2329 y Fh(n)p Fj(\000)p Fi(1)1042 2322 y FC(+)i Fv(D)p FC([)p Fv(0)p FC(])o(\))p Fw(;)e(:::;)189 2414 y FC(\()p Fw(ty)r(pe)291 2421 y Fi(0)310 2414 y Fw(;)g(disp)415 2421 y Fi(0)444 2414 y FC(+)i Fv(D)p FC([)p Fv(0)p FC(])f(+)i(\()p Fv(B)p FC([)p Fv(0)p FC(])e FA(\000)h FC(1\))g FA(\001)f Fw(extent)p FC(\))p Fw(;)f(:::;)g FC(\()p Fw(ty)r(pe)1178 2421 y Fh(n)p Fj(\000)p Fi(1)1245 2414 y Fw(;)g(disp)1350 2421 y Fh(n)p Fj(\000)p Fi(1)1428 2414 y FC(+)i Fv(D)p FC([)p Fv(0)p FC(])f(+)i(\()p Fv(B)p FC([)p Fv(0)p FC(])e FA(\000)h FC(1\))g FA(\001)f Fw(extent)p FC(\))p Fw(;)f(:::;)189 2507 y FC(\()p Fw(ty)r(pe)291 2514 y Fi(0)310 2507 y Fw(;)g(disp)415 2514 y Fi(0)444 2507 y FC(+)i Fv(D)p FC([)p Fv(count)h FA(\000)g Fv(1)p FC(])o(\))p Fw(;)d(:::;)g FC(\()p Fw(ty)r(pe)933 2514 y Fh(n)p Fj(\000)p Fi(1)999 2507 y Fw(;)g(disp)1104 2514 y Fh(n)p Fj(\000)p Fi(1)1181 2507 y FC(+)i Fv(D)p FC([)p Fv(count)h FA(\000)g Fv(1)p FC(])o(\))p Fw(;)d(:::;)189 2600 y FC(\()p Fw(ty)r(pe)291 2607 y Fi(0)310 2600 y Fw(;)g(disp)415 2607 y Fi(0)444 2600 y FC(+)i Fv(D)p FC([)p Fv(count)h FA(\000)g Fv(1)p FC(])e(+)i(\()p Fv(B)p FC([)p Fv(count)g FA(\000)f Fv(1)p FC(])g FA(\000)g FC(1\))f FA(\001)h Fw(extent)p FC(\))p Fw(;)e(:::;)189 2692 y FC(\()p Fw(ty)r(pe)291 2699 y Fh(n)p Fj(\000)p Fi(1)359 2692 y Fw(;)g(disp)464 2699 y Fh(n)p Fj(\000)p Fi(1)542 2692 y FC(+)i Fv(D)p FC([)p Fv(count)h FA(\000)f Fv(1)p FC(])g(+)g(\()p Fv(B)p FC([)p Fv(count)h FA(\000)g Fv(1)p FC(])e FA(\000)i FC(1\))e FA(\001)h Fw(extent)p FC(\))p FA(g)p Fw(:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 51 59 51 58 bop 75 -100 a Fs(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(51)75 45 y Fv(Struct)49 b Fo(MPI)p 315 45 15 2 v 17 w(TYPE)p 428 45 V 16 w(STRUCT)15 b FC(is)h(the)f(most)g(general) h(constructor:)j(It)c(further)h(generalizes)h(the)e(previ-)75 102 y(ous)g(one)g(in)h(that)f(it)g(allo)o(ws)h(eac)o(h)f(blo)q(c)o(k)h (to)e(consist)i(of)f(replications)i(of)d(a)h(di\013eren)o(t)h(datat)o (yp)q(e.)75 205 y Fv(MPI)p 160 205 14 2 v 16 w(TYPE)p 293 205 V 17 w(STRUCT\(count,)c(a)o(rra)o(y)p 729 205 V 15 w(of)p 781 205 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1151 205 V 14 w(of)p 1202 205 V 16 w(displacements,)g(a)o(rra)o(y)p 1599 205 V 15 w(of)p 1651 205 V 15 w(t)o(yp)q(es,)g(new-)75 262 y(t)o(yp)q(e\))117 339 y Fq(IN)155 b Fv(count)482 b Fq(n)o(um)o(b)q(er)18 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))g({)g (also)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)905 395 y(in)c(arra)o(ys)h Fl(a)o(rra)o(y)p 1167 395 13 2 v 15 w(of)p 1216 395 V 14 w(t)o(yp)q(es)p Fq(,)h Fl(a)o(rra)o(y)p 1432 395 V 15 w(of)p 1481 395 V 14 w(displacements)g Fq(and)e Fl(a)o(r-)905 452 y(ra)o(y)p 959 452 V 15 w(of)p 1008 452 V 15 w(blo)q(cklengths)117 524 y Fq(IN)155 b Fv(a)o(rra)o(y)p 416 524 14 2 v 15 w(of)p 468 524 V 16 w(blo)q(cklength)208 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in) o(teger\))117 596 y(IN)155 b Fv(a)o(rra)o(y)p 416 596 V 15 w(of)p 468 596 V 16 w(displacements)164 b Fq(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o (teger\))117 668 y(IN)155 b Fv(a)o(rra)o(y)p 416 668 V 15 w(of)p 468 668 V 16 w(t)o(yp)q(es)327 b Fq(t)o(yp)q(e)20 b(of)f(elemen)o(ts)g(in)g(eac)o(h)h(blo)q(c)o(k)f(\(arra)o(y)g(of)g (handles)g(to)905 724 y(datat)o(yp)q(e)14 b(ob)r(jects\))117 796 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fq(new)15 b(datat)o(yp)q(e)f (\(handle\))75 920 y Fo(int)23 b(MPI)p 245 920 15 2 v 17 w(Type)p 358 920 V 17 w(struct\(int)f(count,)h(int)h(*array)p 1044 920 V 16 w(of)p 1108 920 V 17 w(blocklengths,)393 977 y(MPI)p 468 977 V 17 w(Aint)f(*array)p 748 977 V 17 w(of)p 813 977 V 17 w(displacements,)e(MPI)p 1259 977 V 17 w(Datatype)i(*array)p 1635 977 V 16 w(of)p 1699 977 V 17 w(types,)393 1033 y(MPI)p 468 1033 V 17 w(Datatype)g (*newtype\))75 1120 y(MPI)p 150 1120 V 17 w(TYPE)p 263 1120 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 1120 V 17 w(OF)p 798 1120 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 1120 V 16 w(OF)p 1333 1120 V 17 w(DISPLACEMENTS,)393 1176 y(ARRAY)p 516 1176 V 17 w(OF)p 581 1176 V 17 w(TYPES,)h(NEWTYPE,)f(IERROR\))170 1233 y(INTEGER)h(COUNT,)g(ARRAY)p 651 1233 V 17 w(OF)p 716 1233 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 1233 V 17 w(OF)p 1323 1233 V 17 w(DISPLACEMENTS\(*\),)170 1289 y(ARRAY)p 293 1289 V 17 w(OF)p 358 1289 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)166 1376 y FC(Example:)d(Let)c Fv(t)o(yp)q(e1)g FC(ha)o(v)o(e)f(t)o(yp)q(e)g(map)189 1453 y FA(f)p FC(\()p Fo(double)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(8\))p FA(g)p Fw(;)75 1531 y FC(with)24 b(exten)o(t)g(16.)45 b(Let)24 b Fo(D)g(=)g(\(0,)f(16,)g(26\))p FC(,)j Fo(B)d(=)h(\(2,)g(1,)f (3\))h FC(and)g Fo(T)f(=)h(\(MPI)p 1551 1531 V 17 w(FLOAT,)f(type1,)75 1588 y(MPI)p 150 1588 V 17 w(CHAR\))p FC(.)13 b(Then)i(a)f(call)h(to)f Fv(MPI)p 687 1588 14 2 v 16 w(TYPE)p 820 1588 V 17 w(STRUCT\(3,)g(B,)g (D,)g(T,)g(newt)o(yp)q(e\))i FC(returns)e(datat)o(yp)q(e)g(with)75 1644 y(t)o(yp)q(e)h(map,)189 1722 y FA(f)p FC(\()p Fo(float)n Fw(;)8 b FC(0\))p Fw(;)g FC(\()p Fo(float)m Fw(;)g FC(4\))p Fw(;)g FC(\()p Fo(double)m Fw(;)g FC(16\))p Fw(;)g FC(\()p Fo(char)l Fw(;)g FC(24\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(26\))p Fw(;)g FC(\()p Fo(char)m Fw(;)g FC(27\))p Fw(;)g FC(\()p Fo(char)l Fw(;)g FC(28\))p FA(g)j FC(:)75 1799 y(t)o(w)o(o)f(copies)j(of)e Fv(MPI)p 421 1799 V 15 w(FLO)o(A)l(T)h FC(starting)f(at)g(0,)g(follo)o(w)o(ed)h(b)o(y)f(one)h(cop)o(y)f(of)g Fv(t)o(yp)q(e1)i FC(starting)e(at)f(16,)i(follo)o(w)o(ed)75 1856 y(b)o(y)i(three)g(copies)h(of)e Fv(MPI)p 520 1856 V 16 w(CHAR)p FC(,)i(starting)e(at)g(26.)19 b(\(W)l(e)14 b(assume)g(that)f(a)g(\015oat)h(o)q(ccupies)h(four)f(b)o(ytes.\))166 1912 y(In)i(general,)f(let)h Fv(T)f FC(b)q(e)h(the)f Fv(a)o(rra)o(y)p 742 1912 V 14 w(of)p 793 1912 V 16 w(t)o(yp)q(es)i FC(argumen)o(t,)d(where)31 b Fv(T[i])15 b FC(is)h(a)f(handle)h(to)189 1990 y Fw(ty)r(pe)273 1997 y Fh(i)300 1990 y FC(=)d FA(f)p FC(\()p Fw(ty)r(pe)473 1971 y Fh(i)473 2001 y Fi(0)492 1990 y Fw(;)8 b(disp)597 1971 y Fh(i)597 2001 y Fi(0)616 1990 y FC(\))p Fw(;)g(:::;)g FC(\()o Fw(ty)r(pe)816 1971 y Fh(i)816 2001 y(n)837 2006 y Fe(i)848 2001 y Fj(\000)p Fi(1)895 1990 y Fw(;)g(disp)1000 1971 y Fh(i)1000 2001 y(n)1021 2006 y Fe(i)1034 2001 y Fj(\000)p Fi(1)1081 1990 y FC(\))p FA(g)p Fw(;)75 2068 y FC(with)i(exten)o(t)g Fw(extent)434 2075 y Fh(i)450 2068 y FC(.)18 b(Let)10 b Fv(B)h FC(b)q(e)g(the)f Fv(a)o(rra)o(y)p 826 2068 V 14 w(of)p 877 2068 V 16 w(blo)q(cklength)i FC(argumen)o(t)e(and)g Fv(D)g FC(b)q(e)h(the)f Fv(a)o(rra)o(y)p 1671 2068 V 15 w(of)p 1723 2068 V 16 w(displacements)75 2124 y FC(argumen)o(t.)19 b(Then)c(the)g(newly)h(created)e(datat)o(yp)q(e)h(has)f(a)h(t)o(yp)q(e) g(map)f(with)1394 2092 y Fg(P)1438 2105 y Ff(count)n Fj(\000)p Fi(1)1438 2137 y Fh(i)p Fi(=0)1591 2124 y Fv(B)p FC([)p Fv(i)p FC(])9 b FA(\001)f Fw(n)1715 2131 y Fh(i)1745 2124 y FC(en)o(tries)189 2208 y FA(f)p FC(\()p Fw(ty)r(pe)314 2189 y Fi(0)314 2219 y(0)333 2208 y Fw(;)g(disp)438 2189 y Fi(0)438 2219 y(0)467 2208 y FC(+)i Fv(D)p FC([)p Fv(0)p FC(])o(\))p Fw(;)e(:::;)g FC(\()p Fw(ty)r(pe)794 2189 y Fi(0)794 2219 y Fh(n)815 2224 y Fz(0)832 2208 y Fw(;)g(disp)937 2189 y Fi(0)937 2219 y Fh(n)958 2224 y Fz(0)986 2208 y FC(+)j Fv(D)p FC([)p Fv(0)p FC(])o(\))p Fw(;)d(:::;)189 2292 y FC(\()p Fw(ty)r(pe)291 2273 y Fi(0)291 2303 y(0)310 2292 y Fw(;)g(disp)415 2273 y Fi(0)415 2303 y(0)444 2292 y FC(+)i Fv(D)p FC([)p Fv(0)p FC(])f(+)i(\()p Fv(B)p FC([)p Fv(0)p FC(])e FA(\000)h FC(1\))g FA(\001)f Fw(extent)977 2299 y Fi(0)999 2292 y FC(\))p Fw(;)f(:::;)g FC(\()o Fw(ty)r(pe)1199 2273 y Fi(0)1199 2303 y Fh(n)1220 2308 y Fz(0)1237 2292 y Fw(;)g(disp)1342 2273 y Fi(0)1342 2303 y Fh(n)1363 2308 y Fz(0)1392 2292 y FC(+)i Fv(D)p FC([)p Fv(0)p FC(])f(+)i(\()p Fv(B)p FC([)p Fv(0)p FC(])e FA(\000)h Fv(1)p FC(\))g FA(\001)g Fw(extent)1926 2299 y Fi(0)1947 2292 y FC(\))p Fw(;)e(:::;)189 2376 y FC(\()p Fw(ty)r(pe)291 2357 y Ff(count)n Fj(\000)p Fi(1)291 2388 y(0)436 2376 y Fw(;)g(disp)541 2357 y Ff(count)m Fj(\000)p Fi(1)541 2388 y(0)695 2376 y FC(+)i Fv(D)p FC([)p Fv(count)h FA(\000)g Fv(1)p FC(])o(\))p Fw(;)d(:::;)g FC(\()p Fw(ty)r(pe)1184 2357 y Ff(co)o(unt)l Fj(\000)p Fi(1)1184 2388 y Fh(n)1205 2397 y Ff(cou)o(nt)m Fd(\000)p Fz(1)1342 2388 y Fj(\000)p Fi(1)1389 2376 y Fw(;)g(disp)1494 2357 y Ff(count)m Fj(\000)p Fi(1)1494 2388 y Fh(n)1515 2397 y Ff(count)m Fd(\000)p Fz(1)1653 2388 y Fj(\000)p Fi(1)1711 2376 y FC(+)i Fv(D)p FC([)p Fv(count)h FA(\000)f Fv(1)p FC(]\))p Fw(;)e(:::;)189 2473 y FC(\()p Fw(ty)r(pe)291 2454 y Ff(count)n Fj(\000)p Fi(1)291 2485 y(0)436 2473 y Fw(;)g(disp)541 2454 y Ff(count)m Fj(\000)p Fi(1)541 2485 y(0)695 2473 y FC(+)i Fv(D)p FC([)p Fv(count)h FA(\000)g Fv(1)p FC(])e(+)i(\()p Fv(B)p FC([)p Fv(count)g FA(\000)f Fv(1)p FC(])g FA(\000)g FC(1\))g FA(\001)f Fw(extent)1552 2480 y Ff(count)p Fj(\000)p Fi(1)1699 2473 y FC(\))p Fw(;)f(:::;)189 2556 y FC(\()p Fw(ty)r(pe)291 2537 y Ff(count)n Fj(\000)p Fi(1)291 2569 y Fh(n)312 2578 y Ff(count)n Fd(\000)p Fz(1)451 2569 y Fj(\000)p Fi(1)499 2556 y Fw(;)g(disp)604 2537 y Ff(count)l Fj(\000)p Fi(1)604 2569 y Fh(n)625 2578 y Ff(count)m Fd(\000)p Fz(1)763 2569 y Fj(\000)p Fi(1)820 2556 y FC(+)j Fv(D)p FC([)p Fv(count)g FA(\000)f Fv(1)p FC(])g(+)g(\()p Fv(B)p FC([)p Fv(count)h FA(\000)f Fv(1)p FC(])g FA(\000)g Fv(1)p FC(\))g FA(\001)g Fw(extent)1678 2563 y Ff(count)o Fj(\000)p Fi(1)1824 2556 y FC(\))p FA(g)166 2647 y FC(A)15 b(call)h(to)f Fv(MPI)p 439 2647 V 16 w(TYPE)p 572 2647 V 16 w(HINDEXED\()g(count,)h(B,)f(D,)g(oldt)o(yp)q(e,)h(newt)o(yp)q (e\))g FC(is)g(equiv)m(alen)o(t)h(to)d(a)h(call)75 2704 y(to)d Fv(MPI)p 213 2704 V 16 w(TYPE)p 346 2704 V 17 w(STRUCT\()h(count,)i(B,)e(D,)f(T,)h(newt)o(yp)q(e\))p FC(,)h(where)f(eac)o(h)g(en)o(try)g(of)g Fv(T)g FC(is)g(equal)h(to)e Fv(oldt)o(yp)q(e)p FC(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 52 60 52 59 bop 75 -100 a FC(52)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(3.13.2)49 b(Address)16 b(and)g(extent)f(functions)75 143 y FC(The)g(displacemen)o (ts)i(in)e(a)g(general)h(datat)o(yp)q(e)e(are)h(relativ)o(e)g(to)g (some)f(initial)j(bu\013er)e(address.)20 b Fx(Abso-)75 199 y(lute)15 b(addresses)c FC(can)h(b)q(e)h(substituted)f(for)g(these) g(displacemen)o(ts:)20 b(w)o(e)12 b(treat)f(them)h(as)g(displacemen)o (ts)75 256 y(relativ)o(e)17 b(to)e(\\address)h(zero",)g(the)g(start)f (of)h(the)g(address)h(space.)23 b(This)17 b(initial)h(address)e(zero)g (is)h(indi-)75 312 y(cated)f(b)o(y)g(the)g(constan)o(t)f Fl(MPI)p 601 312 13 2 v 14 w(BOTTOM)p FC(.)g(Th)o(us,)g(a)h(datat)o(yp) q(e)f(can)h(sp)q(ecify)h(the)f(absolute)g(address)g(of)75 368 y(the)e(en)o(tries)g(in)g(the)g(comm)o(unication)g(bu\013er,)g(in)h (whic)o(h)f(case)g(the)g Fv(buf)g FC(argumen)o(t)f(is)h(passed)g(the)g (v)m(alue)75 425 y Fl(MPI)p 152 425 V 14 w(BOTTOM)p FC(.)166 487 y(The)31 b(address)g(of)f(a)h(lo)q(cation)g(in)h(memory)e(can)h(b)q (e)h(found)f(b)o(y)g(in)o(v)o(oking)g(the)g(function)75 544 y Fv(MPI)p 160 544 14 2 v 16 w(ADDRESS)p FC(.)75 654 y Fv(MPI)p 160 654 V 16 w(ADDRESS\(lo)q(cation,)16 b(address\))117 737 y Fq(IN)155 b Fv(lo)q(cation)437 b Fq(lo)q(cation)13 b(in)h(caller)f(memory)f(\(c)o(hoice\))117 824 y(OUT)108 b Fv(address)449 b Fq(address)15 b(of)f(lo)q(cation)f (\(in)o(teger\))75 955 y Fo(int)23 b(MPI)p 245 955 15 2 v 17 w(Address\(void*)f(location,)h(MPI)p 907 955 V 17 w(Aint)g(address\))75 1047 y(MPI)p 150 1047 V 17 w (ADDRESS\(LOCATION,)e(ADDRESS,)i(IERROR\))170 1103 y()g (LOCATION\(*\))170 1160 y(INTEGER)g(ADDRESS,)g(IERROR)166 1252 y FC(Returns)16 b(the)f(\(b)o(yte\))f(address)i(of)e Fv(lo)q(cation)p FC(.)166 1315 y(Example:)147 1457 y Fo(REAL)23 b(A\(100,100\))147 1514 y(INTEGER)f(I1,)i(I2,)f(DIFF)147 1570 y(CALL)g(MPI_ADDRESS\(A\(1,1\),)e(I1,)j(IERROR\))147 1627 y(CALL)f(MPI_ADDRESS\(A\(10,10\),)e(I2,)i(IERROR\))147 1683 y(DIFF)g(=)h(I2)f(-)h(I1)75 1740 y(!)g(The)f(value)g(of)h(DIFF)f (is)h(909*sizeofreal;)e(the)h(values)g(of)h(I1)f(and)h(I2)f(are)75 1796 y(!)h(implementation)e(dependent.)166 1933 y FC(Some)15 b(implemen)o(tations)i(ma)o(y)d(not)h(supp)q(ort)g(calls)h(to)f Fv(MPI)p 1219 1933 14 2 v 16 w(ADDRESS)h FC(in)g(F)l(ortran,)d(where)j (the)75 1989 y Fv(lo)q(cation)g FC(argumen)o(t)e(is)i(of)e(t)o(yp)q(e)i Fv(CHARA)o(CTER)p FC(.)166 2051 y(The)f(follo)o(wing)h(auxiliary)h (functions)f(pro)o(vide)f(useful)i(information)e(on)g(deriv)o(ed)h (datat)o(yp)q(es.)75 2161 y Fv(MPI)p 160 2161 V 16 w(TYPE)p 293 2161 V 17 w(EXTENT\(datat)o(yp)q(e,)g(extent\))117 2244 y Fq(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(\(handle\))117 2332 y(OUT)108 b Fv(extent)471 b Fq(datat)o(yp)q(e)14 b(exten)o(t)h(\(in)o(teger\))75 2462 y Fo(int)23 b(MPI)p 245 2462 15 2 v 17 w(Type)p 358 2462 V 17 w(extent\(MPI)p 615 2462 V 16 w(Datatype)f(datatype,)h(MPI)p 1156 2462 V 17 w(Aint)g(*extent\))75 2555 y(MPI)p 150 2555 V 17 w(TYPE)p 263 2555 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h(IERROR\))170 2611 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)166 2704 y FC(Returns)16 b(the)f(exten)o(t)g(of)g(a)f(datat)o(yp)q(e)h({)g (where)g(exten)o(t)g(is)h(as)f(de\014ned)h(in)g(Eq.)f(3.1.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 53 61 53 60 bop 75 -100 a Fs(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(53)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(SIZE\(datat)o(yp)q(e,)15 b(size\))117 122 y Fq(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(\(handle\))117 196 y(OUT)108 b Fv(size)519 b Fq(datat)o(yp)q(e)14 b(size)h(\(in)o(teger\))75 320 y Fo(int)23 b(MPI)p 245 320 15 2 v 17 w(Type)p 358 320 V 17 w(size\(MPI)p 567 320 V 16 w(Datatype)g(datatype,)f(MPI)p 1108 320 V 17 w(Aint)h(*size\))75 407 y(MPI)p 150 407 V 17 w(TYPE)p 263 407 V 16 w(SIZE\(DATATYPE,)f(SIZE,)i(IERROR\))170 463 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)166 550 y FC(Returns)15 b(the)h(n)o(um)o(b)q(er)f(of)g(b)o(ytes)f(o)q(ccupied)j (b)o(y)e(en)o(tries)h(in)g(the)f(datat)o(yp)q(e)f({)h(i.e.)20 b(the)15 b(sum)g(of)g(the)75 606 y(sizes)h(of)f(the)g(datat)o(yp)q(e)g (elemen)o(ts.)75 710 y Fv(MPI)p 160 710 14 2 v 16 w(TYPE)p 293 710 V 17 w(COUNT\(datat)o(yp)q(e,)i(count\))117 787 y Fq(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(\(handle\))117 861 y(OUT)108 b Fv(count)482 b Fq(datat)o(yp)q(e)14 b(coun)o(t)g(\(in)o(teger\))75 985 y Fo(int)23 b(MPI)p 245 985 15 2 v 17 w(Type)p 358 985 V 17 w(count\(MPI)p 591 985 V 16 w(Datatype)g(datatype,)f(int)i(*count\))75 1072 y(MPI)p 150 1072 V 17 w(TYPE)p 263 1072 V 16 w(COUNT\(DATATYPE,)e (COUNT,)h(IERROR\))170 1128 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)166 1215 y FC(Returns)16 b(the)f(n)o(um)o(b)q(er)g(of)g(en)o(tries)h(in)g (the)f(datat)o(yp)q(e.)75 1335 y Fp(3.13.3)49 b(Lo)o(w)o(er-b)q(ound)17 b(and)g(upp)q(er-b)q(ound)g(ma)o(rk)o(ers)75 1421 y FC(It)e(is)g(often) g(con)o(v)o(enien)o(t)g(to)g(de\014ne)h(explicitly)h(the)e(lo)o(w)o(er) g(b)q(ound)h(and)f(upp)q(er)h(b)q(ound)f(of)g(a)g(t)o(yp)q(e)g(map,)75 1477 y(and)d(o)o(v)o(erwrite)e(the)i(de\014nition)h(giv)o(en)f(b)o(y)f (Equation)g(3.1.)18 b(This)12 b(allo)o(ws)f(to)g(de\014ne)i(a)e(datat)o (yp)q(e)f(that)h(has)75 1534 y(\\holes")k(at)f(its)h(b)q(eginning)i(or) d(its)h(end;)h(or)e(a)h(datat)o(yp)q(e)f(with)h(en)o(tries)g(that)f (extend)i(ab)q(o)o(v)o(e)e(the)h(upp)q(er)75 1590 y(b)q(ound)h(or)f(b)q (elo)o(w)h(the)f(lo)o(w)o(er)g(b)q(ound.)21 b(Examples)16 b(of)f(suc)o(h)g(a)g(usage)g(are)g(pro)o(vided)h(in)g(section)g (3.13.6.)75 1646 y(T)l(o)f(ac)o(hiev)o(e)h(this,)g(w)o(e)f(add)g(t)o(w) o(o)f(additional)j(\\pseudo-datat)o(yp)q(es")e Fv(MPI)p 1349 1646 14 2 v 16 w(LB)p FC(,)g(and)g Fv(MPI)p 1618 1646 V 16 w(UB)h FC(that)f(can)75 1703 y(b)q(e)j(used,)g(resp)q(ectiv)o (ely)l(,)h(to)e(mark)f(the)i(lo)o(w)o(er)f(b)q(ound)h(or)f(the)g(upp)q (er)h(b)q(ound)g(of)f(a)g(datat)o(yp)q(e.)26 b(These)75 1759 y(pseudo-datat)o(yp)q(es)16 b(o)q(ccup)o(y)g(no)f(space)h(\()p Fw(extent)p FC(\()p Fv(MPI)p 1022 1759 V 16 w(LB)p FC(\))d(=)h Fw(extent)p FC(\()p Fv(MPI)p 1400 1759 V 16 w(UB)q FC(\))f(=)g(0\).)20 b(They)c(do)f(not)75 1816 y(a\013ect)d(the)h(size)h(or)e(coun)o(t)h(of) f(a)h(datat)o(yp)q(e,)f(and)h(do)g(not)f(a\013ect)g(the)h(the)g(con)o (ten)o(t)g(of)f(a)h(message)f(created)75 1872 y(with)k(this)f(datat)o (yp)q(e.)20 b(Ho)o(w)o(ev)o(er,)13 b(they)j(do)f(a\013ect)f(the)i (de\014nition)h(of)e(the)g(exten)o(t)g(of)g(a)f(datat)o(yp)q(e)h(and,) 75 1929 y(therefore,)g(a\013ect)f(the)h(outcome)g(of)g(a)g(replication) i(of)d(this)i(datat)o(yp)q(e)f(b)o(y)g(a)g(datat)o(yp)q(e)f (constructor.)166 1985 y(Example:)30 b(Let)21 b Fo(D)j(=)f(\(-3,)h(0,)f (6\))p FC(;)f Fo(T)i(=)g(\(MPI)p 1033 1985 15 2 v 16 w(LB,)g(MPI)p 1217 1985 V 17 w(INT,)f(MPI)p 1425 1985 V 17 w(UB\))p FC(,)c(and)h Fo(B)k(=)g(\(1,)f(1,)75 2042 y(1\))p FC(.)c(Then)d(a)e(call)i(to)f Fv(MPI)p 535 2042 14 2 v 15 w(TYPE)p 667 2042 V 17 w(STRUCT\(3,)g(B,)g(D,)f(T,)g(t)o(yp)q (e1\))h FC(creates)g(a)g(new)g(datat)o(yp)q(e)f(that)g(has)75 2098 y(an)e(exten)o(t)f(of)g(9)h(\(from)e(-3)i(to)f(5\),)g(and)h(con)o (tains)g(an)g(in)o(teger)g(at)f(displacemen)o(t)i(0.)18 b(This)12 b(is)h(the)e(datat)o(yp)q(e)75 2155 y(de\014ned)j(b)o(y)e (the)g(sequence)i FA(f)p Fo(\(lb,)23 b(-3\),)g(\(int,)g(0\),)h(\(ub,)f (6\))p FA(g)g FC(.)c(If)13 b(this)g(t)o(yp)q(e)f(is)h(replicated)h(t)o (wice)75 2211 y(b)o(y)h(a)g(call)h(to)f Fv(MPI)p 400 2211 V 15 w(TYPE)p 532 2211 V 17 w(CONTIGUOUS\(2,)h(t)o(yp)q(e1,)g(t)o (yp)q(e2\))f FC(then)h(the)f(newly)h(created)f(t)o(yp)q(e)g(can)g(b)q (e)75 2267 y(describ)q(ed)e(b)o(y)d(the)h(sequence)h FA(f)p Fo(\(lb,)23 b(-3\),)g(\(int,)g(0\),)h(\(int,9\),)e(\(ub,)i(15\)) p FA(g)f FC(.)18 b(\(En)o(tries)11 b(of)f(t)o(yp)q(e)75 2324 y Fo(lb)15 b FC(or)g Fo(ub)f FC(can)i(b)q(e)g(deleted)g(if)g(they) f(are)g(not)g(at)f(the)i(endp)q(oin)o(ts)g(of)f(the)g(datat)o(yp)q (e.\))166 2380 y(In)h(general,)f(if)189 2467 y Fw(T)6 b(y)r(pemap)12 b FC(=)h FA(f)p FC(\()p Fw(ty)r(pe)562 2474 y Fi(0)581 2467 y Fw(;)8 b(disp)686 2474 y Fi(0)705 2467 y FC(\))p Fw(;)g(:::;)g FC(\()o Fw(ty)r(pe)905 2474 y Fh(n)p Fj(\000)p Fi(1)971 2467 y Fw(;)g(disp)1076 2474 y Fh(n)p Fj(\000)p Fi(1)1144 2467 y FC(\))p FA(g)p Fw(;)75 2553 y FC(then)16 b(the)f Fx(lo)o(w)o(er)h(b)q(ound)g FC(of)f Fw(T)6 b(y)r(pemap)15 b FC(is)g(de\014ned)i(to)e(b)q(e)189 2669 y Fw(l)q(b)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))k(=)506 2597 y Fg(\()560 2641 y FC(min)636 2648 y Fh(j)662 2641 y Fw(disp)746 2648 y Fh(j)1119 2641 y FC(if)16 b(no)f(en)o(try)g(has)g (basic)h(t)o(yp)q(e)f Fo(lb)560 2697 y FC(min)q FA(f)p Fw(disp)743 2704 y Fh(j)788 2697 y FC(:)28 b Fw(ty)r(pe)913 2704 y Fh(j)944 2697 y FC(=)13 b Fo(lb)p FA(g)41 b FC(otherwise)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 54 62 54 61 bop 75 -100 a FC(54)613 b Fs(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(Similarly)l(,)h (the)e Fx(upp)q(er)j(b)q(ound)e FC(of)e Fw(T)6 b(y)r(pemap)15 b FC(is)h(de\014ned)h(to)d(b)q(e)189 182 y Fw(ub)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))11 b(=)518 110 y Fg(\()572 154 y FC(max)657 161 y Fh(j)682 154 y Fw(disp)766 161 y Fh(j)794 154 y FC(+)g Fw(siz)r(eof)5 b FC(\()p Fw(ty)r(pe)1072 161 y Fh(j)1090 154 y FC(\))57 b(if)15 b(no)h(en)o(try)e(has)h(basic)h (t)o(yp)q(e)g Fo(ub)572 210 y FC(max)p FA(f)p Fw(disp)764 217 y Fh(j)809 210 y FC(:)27 b Fw(ty)r(pe)933 217 y Fh(j)964 210 y FC(=)13 b Fo(ub)p FA(g)67 b FC(otherwise)166 316 y(Then)189 423 y Fw(extent)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))13 b(=)g Fw(ub)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))i FA(\000)j Fw(l)q(b)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))i(+)i Fw(\017)75 527 y FC(If)19 b Fw(ty)r(pe)208 534 y Fh(i)242 527 y FC(requires)h(alignmen)o(t)g(to)e(a)h(b)o(yte)g(address)g(that)g (is)h(a)e(m)o(ultiple)k(of)c Fw(k)1460 534 y Fh(i)1474 527 y FC(,)i(then)f Fw(\017)h FC(is)g(the)f(least)75 583 y(nonnegativ)o(e)d(incremen)o(t)g(needed)g(to)f(round)g Fw(extent)p FC(\()p Fw(T)6 b(y)r(pemap)p FC(\))16 b(to)e(the)h(next)h (m)o(ultiple)h(of)d(max)1794 590 y Fh(i)1816 583 y Fw(k)1840 590 y Fh(i)1854 583 y FC(.)166 642 y(The)g(formal)f(de\014nitions)j (giv)o(en)e(for)f(the)h(v)m(arious)g(datat)o(yp)q(e)f(constructors)g (apply)i(no)o(w,)e(with)h(the)75 699 y(amended)i(de\014nition)h(of)e Fx(exten)o(t)p FC(.)75 835 y Fp(3.13.4)49 b(Commit)16 b(and)h(free)75 926 y FC(A)g(datat)o(yp)q(e)f(ob)s(ject)h(has)g(to)f(b) q(e)h Fx(committed)h FC(b)q(efore)f(it)g(can)g(b)q(e)h(used)g(in)f(a)g (comm)o(unication.)26 b(The)75 982 y(system)20 b(ma)o(y)g(\\compile")i (at)e(commit)g(time)h(an)g(in)o(ternal)g(represen)o(tation)g(for)f(the) h(datat)o(yp)q(e)f(that)75 1039 y(facilitates)c(comm)o(unication,)g (e.g.)k(c)o(hange)15 b(from)g(a)g(compacted)g(represen)o(tation)h(to)e (a)h(\015at)g(represen-)75 1095 y(tation)j(of)g(the)g(datat)o(yp)q(e,)g (and)g(select)h(the)f(most)g(con)o(v)o(enien)o(t)g(transfer)g(mec)o (hanism.)29 b(A)19 b(commited)75 1152 y(datat)o(yp)q(e)c(can)g(still)i (b)q(e)e(used)h(as)f(a)g(argumen)o(t)f(in)i(datat)o(yp)q(e)f (constructors.)166 1211 y(Note:)28 b(the)20 b(commit)f(op)q(eration)h (commits)f(the)h(datat)o(yp)q(e,)f(i.e.,)i(the)e(formal)g(description)i (of)e(a)75 1267 y(comm)o(unication)i(bu\013er,)h(not)e(the)h(con)o(ten) o(t)f(of)g(that)g(bu\013er.)36 b(Th)o(us,)22 b(after)e(a)g(datat)o(yp)q (e)g(has)h(b)q(een)75 1324 y(committed,)d(it)h(can)f(b)q(e)h(rep)q (eatedly)g(reused)g(to)e(comm)o(unicate)h(the)h(c)o(hanging)f(con)o (ten)o(t)g(of)f(a)h(bu\013er)75 1380 y(or,)c(indeed,)j(the)e(con)o(ten) o(t)g(of)g(di\013eren)o(t)g(bu\013ers,)g(with)g(di\013eren)o(t)h (starting)e(addresses.)75 1486 y Fv(MPI)p 160 1486 14 2 v 16 w(TYPE)p 293 1486 V 17 w(COMMIT\(datat)o(yp)q(e\))117 1566 y Fq(INOUT)62 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(that)g(is)g(committed)e(\(handle\))75 1693 y Fo(int)23 b(MPI)p 245 1693 15 2 v 17 w(Type)p 358 1693 V 17 w(commit\(MPI)p 615 1693 V 16 w(Datatype)f(*datatype\))75 1782 y(MPI)p 150 1782 V 17 w(TYPE)p 263 1782 V 16 w(COMMIT\(DATATYPE,)g(IERROR\))170 1839 y(INTEGER)h(DATATYPE,)g(IERROR)166 1928 y FC(There)15 b(is)h(no)f(need)h(to)f(commit)g(basic)h(datat)o(yp)q(es;)e(they)h(are) g(\\pre-committed".)75 2034 y Fv(MPI)p 160 2034 14 2 v 16 w(TYPE)p 293 2034 V 17 w(FREE\(datat)o(yp)q(e\))117 2114 y Fq(INOUT)62 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(that)g(is)g(freed)h(\(handle\))75 2241 y Fo(int)23 b(MPI)p 245 2241 15 2 v 17 w(Type)p 358 2241 V 17 w(free\(MPI)p 567 2241 V 16 w(Datatype)g(*datatype\))75 2330 y(MPI)p 150 2330 V 17 w(TYPE)p 263 2330 V 16 w(FREE\(DATATYPE,)f(IERROR\))170 2386 y(INTEGER)h(DATATYPE,)g(IERROR)166 2475 y FC(Marks)16 b(the)h(datat)o(yp)q(e)f(ob)s(ject)g(asso)q(ciated)h(with)g Fv(datat)o(yp)q(e)i FC(for)d(deallo)q(cation.)26 b(The)17 b(ob)s(ject)f(will)75 2532 y(b)q(e)g(deallo)q(cated)h(after)d(an)o(y)i (p)q(ending)h(comm)o(unication)f(that)e(uses)i(this)g(ob)s(ject)f (completes,)g(at)g(whic)o(h)75 2588 y(p)q(oin)o(t)21 b Fv(datat)o(yp)q(e)g FC(b)q(ecomes)g(n)o(ull.)36 b(It)20 b(is)h(erroneous)f(to)f(use)i(a)f(datat)o(yp)q(e)f(as)h(argumen)o(t)f (to)h(an)g(MPI)75 2645 y(function)c(after)e(it)i(w)o(as)e(freed.)166 2704 y(Example:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 55 63 55 62 bop 75 -100 a Fs(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(55)75 102 y Fo(INTEGER)23 b(type1,)g(type2)75 158 y(CALL)g(MPI_TYPE_CONTIGUOUS\(5,)e(MPI_REAL,)i(type1,)g(ierr\))409 214 y(!)h(new)f(type)h(object)f(created)75 271 y(CALL)g (MPI_TYPE_COMMIT\(type1,)e(ierr\))409 327 y(!)j(now)f(type1)g(can)h(be) f(used)h(for)f(communication)75 384 y(type2)g(=)h(type1)409 440 y(!)g(type2)f(can)g(be)h(used)f(for)h(communication)409 497 y(!)g(\(it)f(is)h(a)g(handle)f(to)g(same)h(object)f(as)g(type1\))75 553 y(CALL)g(MPI_TYPE_VECTOR\(3,)f(5,)h(4,)h(MPI_REAL,)f(type1,)g (ierr\))409 610 y(!)h(new)f(uncommitted)g(type)g(object)g(created)75 666 y(CALL)g(MPI_TYPE_COMMIT\(type1,)e(ierr\))409 723 y(!)j(now)f(type1)g(can)h(be)f(used)h(anew)f(for)h(communication)166 835 y FC(F)l(reeing)19 b(a)f(datat)o(yp)q(e)f(do)q(es)i(not)f(a\013ect) f(an)o(y)h(other)g(datat)o(yp)q(e)f(that)h(w)o(as)f(built)j(from)d(the) i(freed)75 891 y(datat)o(yp)q(e.)34 b(The)20 b(system)g(should)h(b)q (eha)o(v)o(e)g(as)e(if)i(a)f(separate)f(cop)o(y)h(of)g(the)g(datat)o (yp)q(e)g(w)o(as)f(created)75 947 y(whenev)o(er)d(this)f(datat)o(yp)q (e)g(w)o(as)f(incorp)q(orated)i(in)g(a)f(new)g(deriv)o(ed)i(datat)o(yp) q(e.)166 1081 y Fn(Implemen)o(tati)o(on)12 b(note:)166 1132 y Fq(The)e(implemen)o(tati)o(on)d(ma)o(y)g(k)o(eep)j(a)f (reference)j(coun)o(t)e(of)f(activ)o(e)g(comm)o(unications)d(that)k (use)g(the)g(datat)o(yp)q(e,)75 1182 y(in)15 b(order)i(to)f(decide)h (when)f(to)g(free)h(it.)24 b(Also,)15 b(one)h(ma)o(y)e(implemen)o(t)g (constructors)k(of)d(deriv)o(ed)h(datat)o(yp)q(es)h(so)75 1232 y(that)e(they)h(k)o(eep)g(p)q(oin)o(ters)g(to)f(their)h(datat)o (yp)q(e)f(argumen)o(ts,)f(rather)j(then)e(cop)o(ying)g(them.)21 b(In)16 b(suc)o(h)g(case,)g(one)75 1281 y(needs)e(to)e(k)o(eep)h(trac)o (k)f(of)g(activ)o(e)g(datat)o(yp)q(e)g(de\014nition)g(references)j(in)d (order)h(to)f(kno)o(w)g(when)h(a)f(datat)o(yp)q(e)g(ob)r(ject)75 1331 y(can)i(b)q(e)h(freed.)75 1542 y Fp(3.13.5)49 b(Use)16 b(of)h(general)f(datat)o(yp)q(es)g(in)h(communication)75 1630 y FC(Handles)g(to)f(deriv)o(ed)h(datat)o(yp)q(es)f(can)h(b)q(e)f (passed)h(to)f(a)g(comm)o(unication)h(call)g(wherev)o(er)f(a)g(datat)o (yp)q(e)75 1687 y(argumen)o(t)10 b(is)g(required.)20 b(A)10 b(call)i(of)e(the)g(form)g Fv(MPI)p 940 1687 14 2 v 16 w(SEND\(buf,)h(count,)i(datat)o(yp)q(e)f(,)f(...\))p FC(,)e(where)i Fo(count)h Fw(>)75 1743 y Fo(1)p FC(,)g(is)h(in)o (terpreted)g(as)f(if)h(the)g(call)g(w)o(as)f(passed)h(a)f(new)g(datat)o (yp)q(e)g(whic)o(h)i(is)f(the)f(concatenation)h(of)f Fv(count)75 1799 y FC(copies)k(of)f Fv(datat)o(yp)q(e)p FC(.)21 b(Th)o(us,)166 1857 y Fv(MPI)p 251 1857 V 16 w(SEND\(buf,)15 b(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm\))e FC(is)j(equiv)m(alen)o(t)h(to)75 1970 y Fo(MPI_TYPE_CONTIGUOUS\(count,) j(datatype,)j(newtype\))75 2027 y(MPI_TYPE_COMMIT\(newtype\))75 2083 y(MPI_SEND\(buf,)f(1,)i(newtype,)e(dest,)i(tag,)f(comm\))166 2195 y FC(The)14 b(same)g(applies)h(to)f(all)h(other)e(comm)o (unication)i(functions)g(that)e(ha)o(v)o(e)g(a)h Fv(count)h FC(and)f Fv(datat)o(yp)q(e)75 2251 y FC(argumen)o(t.)166 2309 y(Let)h Fv(datat)o(yp)q(e)i FC(ha)o(v)o(e)e(t)o(yp)q(e)g(map)189 2422 y FA(f)p FC(\()p Fw(ty)r(pe)314 2429 y Fi(0)333 2422 y Fw(;)8 b(disp)438 2429 y Fi(0)457 2422 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 2429 y Fh(n)p Fj(\000)p Fi(1)723 2422 y Fw(;)h(disp)828 2429 y Fh(n)p Fj(\000)p Fi(1)895 2422 y FC(\))p FA(g)p Fw(;)166 2534 y FC(with)20 b(exten)o(t)f Fw(extent)i FC(\(empt)o(y)e(en)o(tries)g(deleted\).)34 b(Then)20 b(the)g(send)g(bu\013er)f(used)h(b)o(y)g(the)f(send)75 2591 y(op)q(eration)c(consists)h(of)f Fw(n)10 b FA(\002)g Fo(count)15 b FC(en)o(tries,)g(where)h(en)o(try)e Fw(i)c FA(\002)h Fw(n)f FC(+)g Fw(j)18 b FC(is)e(at)e(lo)q(cation)189 2704 y Fw(addr)282 2711 y Fh(i;j)333 2704 y FC(=)f Fo(buf)d FC(+)g Fw(extent)i FA(\002)e Fw(i)g FC(+)h Fw(disp)848 2711 y Fh(j)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 56 64 56 63 bop 75 -100 a FC(56)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)166 45 y FC(and)f(has)f(t)o (yp)q(e)h Fw(ty)r(pe)520 52 y Fh(j)538 45 y FC(,)g(for)f Fw(i)e FC(=)h(0)p Fw(;)8 b(:::;)g Fo(cou)o(nt)e FA(\000)j FC(1)15 b(and)f Fw(j)h FC(=)e(0)p Fw(;)8 b(:::;)g(n)e FA(\000)k FC(1.)19 b(These)c(en)o(tries)g(need)g(not)75 102 y(b)q(e)h(con)o(tiguous,)f(nor)g(distinct;)h(their)f(order)g(can)h (b)q(e)g(arbitrary)l(.)166 158 y(The)g(v)m(ariable)h(stored)e(at)g (address)g Fw(addr)879 165 y Fh(i;j)933 158 y FC(in)i(the)e(calling)j (program)c(should)j(b)q(e)f(of)f(a)g(t)o(yp)q(e)h(that)75 214 y(matc)o(hes)d Fw(ty)r(pe)333 221 y Fh(j)352 214 y FC(,)h(where)g(t)o(yp)q(e)g(matc)o(hing)f(is)i(de\014ned)g(as)e(in)i (section)f(3.5.)19 b(The)14 b(message)f(sen)o(t)h(con)o(tains)75 271 y Fw(n)c FA(\002)h Fo(count)j FC(en)o(tries,)i(where)f(en)o(try)g Fw(i)10 b FA(\002)g Fw(n)g FC(+)h Fw(j)17 b FC(has)e(t)o(yp)q(e)h Fw(ty)r(pe)1159 278 y Fh(j)1177 271 y FC(.)166 327 y(Similarly)l(,)f (supp)q(ose)f(that)e(a)h(receiv)o(e)h(op)q(eration)f Fv(MPI)p 1100 327 14 2 v 16 w(RECV\(buf,)h(count,)g(datat)o(yp)q(e,)h (source,)e(tag,)75 384 y(comm,)g(status\))k FC(is)e(executed,)h(where)g Fv(datat)o(yp)q(e)g FC(has)g(t)o(yp)q(e)f(map)189 472 y FA(f)p FC(\()p Fw(ty)r(pe)314 479 y Fi(0)333 472 y Fw(;)8 b(disp)438 479 y Fi(0)457 472 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 479 y Fh(n)p Fj(\000)p Fi(1)723 472 y Fw(;)h(disp)828 479 y Fh(n)p Fj(\000)p Fi(1)895 472 y FC(\))p FA(g)p Fw(;)166 560 y FC(with)23 b(exten)o(t)f Fw(extent)j FC(\(empt)o(y)d(en)o(tries)h(deleted\).)43 b(Then)24 b(the)f(receiv)o(e)g(bu\013er)g(used)g(for)g(this)75 616 y(receiv)o(e)e(op)q(eration)f(consists)g(of)f(the)h Fw(n)13 b FA(\002)h Fo(count)19 b FC(en)o(tries,)i(where)f(en)o(try)f Fw(i)13 b FA(\002)h Fw(n)f FC(+)h Fw(j)22 b FC(is)e(at)f(lo)q(cation)75 673 y Fo(buf)6 b FC(+)g Fw(extent)g FA(\002)h Fw(i)f FC(+)h Fw(disp)517 680 y Fh(j)551 673 y FC(and)13 b(has)h(t)o(yp)q(e)f Fw(ty)r(pe)901 680 y Fh(j)920 673 y FC(.)19 b(If)14 b(the)f(incoming)i (message)e(consists)g(of)g Fw(k)i FC(elemen)o(ts,)75 729 y(then)i(w)o(e)f(m)o(ust)g(ha)o(v)o(e)g Fw(k)f FA(\024)g Fw(n)d FA(\002)f Fo(count)p FC(;)16 b(the)g Fw(i)11 b FA(\002)g Fw(n)g FC(+)h Fw(j)s FC(-th)j(elemen)o(t)i(of)f(the)h (message)f(should)h(ha)o(v)o(e)f(a)75 786 y(t)o(yp)q(e)f(that)g(matc)o (hes)g Fw(ty)r(pe)536 793 y Fh(j)554 786 y FC(.)166 842 y(Note)c(that)g(t)o(yp)q(e)h(matc)o(hing)f(is)h(de\014ned)h(according)f (to)f(the)h(t)o(yp)q(e)f(signature)h(of)f(the)h(corresp)q(onding)75 898 y(datat)o(yp)q(es,)f(i.e.,)h(the)f(sequence)i(of)e(basic)h(t)o(yp)q (e)g(comp)q(onen)o(ts,)f(and)h(do)q(es)g(not)f(dep)q(end)i(on)e(the)g (datat)o(yp)q(e)75 955 y(de\014nition.)166 1011 y(Example:)75 1092 y Fo(...)75 1149 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\()e(MPI_REAL,)i (2,)h(type2,)f(...\))75 1205 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e (MPI_REAL,)i(4,)h(type4,)f(...\))75 1261 y(CALL)g (MPI_TYPE_CONTIGUOUS\()e(type2,)i(2,)h(type22,)f(...\))75 1318 y(...)75 1374 y(CALL)g(MPI_SEND\()g(a,)h(4,)f(MPI_REAL,)g(...\))75 1431 y(CALL)g(MPI_SEND\()g(a,)h(2,)f(type2,)g(...\))75 1487 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type22,)g(...\))75 1544 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type4,)g(...\))75 1600 y(...)75 1657 y(CALL)g(MPI_RECV\()g(a,)h(4,)f(MPI_REAL,)g(...\))75 1713 y(CALL)g(MPI_RECV\()g(a,)h(2,)f(type2,)g(...\))75 1770 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type22,)g(...\))75 1826 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type4,)g(...\))166 1907 y FC(F)l(rom)14 b(a)h(t)o(yp)q(e)h(matc)o(hing)f(viewp)q(oin)o(t)h (an)o(y)f(of)g(the)g(sends)h(legally)g(matc)o(hes)f(an)o(y)g(of)g(the)g (receiv)o(es.)166 1963 y(A)j(datat)o(yp)q(e)g(used)h(in)g(a)g(receiv)o (e)g(op)q(eration)g(ma)o(y)e(not)h(ha)o(v)o(e)g(o)o(v)o(erlapping)h(en) o(tries)g(in)g(its)g(t)o(yp)q(e)75 2020 y(map:)h(the)15 b(same)g(lo)q(cation)h(in)g(the)f(receiv)o(e)h(bu\013er)f(cannot)g(b)q (e)h(written)g(t)o(wice)f(during)h(a)f(receiv)o(e.)166 2076 y(T)o(yp)q(e)23 b(matc)o(hing)g(requiremen)o(ts)h(imply)g(that)f (\(nonempt)o(y\))f(en)o(tries)i(in)f(the)h(t)o(yp)q(e)f(map)g(of)f(a)75 2133 y(datat)o(yp)q(e)f(cannot)g(o)o(v)o(erlap,)h(unless)h(they)e (coincide)j(and)d(ha)o(v)o(e)g(the)g(same)g(basic)h(t)o(yp)q(e,)h(or)e (one)g(of)75 2189 y(them)e(has)g(basic)h(t)o(yp)q(e)f Fv(b)o(yte)p FC(:)29 b(otherwise,)20 b(when)f(the)g(datat)o(yp)q(e)g (is)h(used)f(in)h(a)f(comm)o(unication,)i(at)75 2245 y(least)14 b(one)f(of)h(the)f(t)o(w)o(o)g(o)o(v)o(erlapping)h(en)o (tries)g(will)h(not)e(b)q(e)i(t)o(yp)q(e-consisten)o(t)f(with)g(the)f (comm)o(unication)75 2302 y(bu\013er.)166 2358 y(Supp)q(ose,)19 b(again,)f(that)f Fv(MPI)p 683 2358 V 15 w(RECV\(buf,)j(count,)f(datat) o(yp)q(e,)g(dest,)g(tag,)f(comm,)e(status\))j FC(is)f(exe-)75 2415 y(cuted,)d(where)h Fv(datat)o(yp)q(e)h FC(has)e(t)o(yp)q(e)g(map) 189 2503 y FA(f)p FC(\()p Fw(ty)r(pe)314 2510 y Fi(0)333 2503 y Fw(;)8 b(disp)438 2510 y Fi(0)457 2503 y FC(\))p Fw(;)g(:::;)f FC(\()p Fw(ty)r(pe)657 2510 y Fh(n)p Fj(\000)p Fi(1)723 2503 y Fw(;)h(disp)828 2510 y Fh(n)p Fj(\000)p Fi(1)895 2503 y FC(\))p FA(g)p Fw(:)166 2591 y FC(The)k(receiv)o(ed)g (message)f(need)h(not)f(\014ll)i(all)g(the)e(receiv)o(e)h(bu\013er,)g (nor)f(do)q(es)h(it)g(need)g(to)f(\014ll)i(a)e(n)o(um)o(b)q(er)75 2647 y(of)i(lo)q(cations)h(whic)o(h)g(is)f(a)g(m)o(ultiple)i(of)e Fw(n)p FC(;)h(an)o(y)f(n)o(um)o(b)q(er)g Fw(k)i FC(of)d(basic)i(elemen) o(ts)g(can)f(b)q(e)h(receiv)o(ed,)h(where)75 2704 y(0)d FA(\024)h Fw(k)h FA(\024)f Fo(count)c FA(\002)i Fw(n)p FC(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 57 65 57 64 bop 75 -100 a Fs(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(57)166 45 y(The)14 b(n)o(um)o(b)q(er)g(of)f(elemen)o(ts)i (receiv)o(ed)g(can)f(b)q(e)g(retriev)o(ed)g(from)f Fv(status)j FC(using)e(the)g(query)g(function)75 102 y Fv(MPI)p 160 102 14 2 v 16 w(GET)p 264 102 V 17 w(ELEMENTS)p FC(:)75 206 y Fv(MPI)p 160 206 V 16 w(GET)p 264 206 V 17 w(ELEMENTS\()h (status,)h(datat)o(yp)q(e,)h(count\))117 283 y Fq(IN)155 b Fv(status)476 b Fq(return)15 b(status)g(of)e(receiv)o(e)i(op)q (eration)f(\(Status\))117 360 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fq(datat)o(yp)q(e)14 b(used)h(b)o(y)f(receiv)o(e)h(op)q(eration)f (\(handle\))117 436 y(OUT)108 b Fv(count)482 b Fq(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(basic)f(elemen)o(ts)g(\(in)o(teger\))75 561 y Fo(int)23 b(MPI)p 245 561 15 2 v 17 w(Get)p 334 561 V 17 w(elements\(MPI)p 639 561 V 16 w(Status)g(status,)f(MPI)p 1084 561 V 17 w(Datatype)h(datatype,)g(int)g(*count\))75 648 y(MPI)p 150 648 V 17 w(GET)p 239 648 V 17 w(ELEMENTS\(STATUS,)e (DATATYPE,)i(COUNT,)g(IERROR\))170 704 y(INTEGER)g(STATUS\(MPI)p 604 704 V 16 w(STATUS)p 764 704 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 791 y FC(The)13 b(previously)h(used)f(function)h Fv(MPI)p 834 791 14 2 v 16 w(GET)p 938 791 V 16 w(COUNT)g FC(has)f(a)f(di\013eren)o(t)h(b)q(eha)o(vior:)19 b(it)13 b(returns)g(the)75 848 y(n)o(um)o(b)q(er)g(of)f(\\top)g(elemen)o(ts")h (receiv)o(ed.)21 b(In)13 b(the)g(previous)g(example,)h Fv(MPI)p 1368 848 V 16 w(GET)p 1472 848 V 16 w(COUNT)g FC(ma)o(y)e(return)75 904 y(an)o(y)h(in)o(teger)g(v)m(alue)i Fw(k)q FC(,)e(where)h(0)e FA(\024)h Fw(k)h FA(\024)f Fo(count)o FC(.)19 b(If)14 b Fv(MPI)p 1053 904 V 15 w(GET)p 1156 904 V 17 w(COUNT)g FC(returns)f Fw(k)q FC(,)h(then)f(the)h(n)o(um) o(b)q(er)75 961 y(of)j(basic)i(elemen)o(ts)f(receiv)o(ed)h(\(and)e(the) h(v)m(alue)h(returned)f(b)o(y)g Fv(MPI)p 1258 961 V 16 w(GET)p 1362 961 V 16 w(ELEMENTS)p FC(\))g(is)g Fw(n)12 b FA(\002)g Fw(k)q FC(.)28 b(If)75 1017 y(the)18 b(n)o(um)o(b)q(er)g (of)f(basic)h(elemen)o(ts)h(receiv)o(ed)g(is)f(not)f(a)h(m)o(ultiple)h (of)e Fw(n)p FC(,)h(i.e.)28 b(if)18 b(the)g(receiv)o(e)h(op)q(eration) 75 1073 y(has)e(not)g(receiv)o(ed)i(an)f(in)o(tegral)f(n)o(um)o(b)q(er) h(of)f Fv(datat)o(yp)q(e)i FC(\\copies",)f(then)g Fv(MPI)p 1441 1073 V 16 w(GET)p 1545 1073 V 17 w(COUNT)g FC(returns)75 1130 y(the)d(v)m(alue)i Fl(MPI)p 348 1130 13 2 v 14 w(UNDEFINED)p FC(.)166 1187 y(Example:)75 1296 y Fo(...)75 1353 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\(2,)e(MPI_REAL,)i(Type2,)g(ierr\))75 1409 y(CALL)g(MPI_COMMIT\(Type2,)f(ierr\))75 1466 y(...)75 1522 y(CALL)h(MPI_RANK\(comm,)f(rank,)h(ierr\))75 1578 y(IF\(rank.EQ.0\))147 1635 y(THEN)218 1691 y(CALL)g(MPI_SEND\(a,)g(2,)g (MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 1748 y(CALL)g(MPI_SEND\(a,)g (3,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))147 1804 y(ELSE)218 1861 y(CALL)g(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g (ierr\))218 1917 y(CALL)g(MPI_GET_COUNT\(stat,)f(i,)h(ierr\))191 b(!)23 b(returns)g(i=1)218 1974 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e(i,)j (ierr\))119 b(!)23 b(returns)g(i=2)218 2030 y(CALL)g(MPI_RECV\(a,)g(2,) g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g(ierr\))218 2087 y(CALL)g(MPI_GET_COUNT\(stat,)f(i,)h(ierr\))191 b(!)23 b(returns)g(i=MPI_UNDEFINED)218 2143 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e (i,)j(ierr\))119 b(!)23 b(returns)g(i=3)75 2199 y(END)g(IF)166 2308 y FC(The)15 b(function)h Fv(MPI)p 522 2308 14 2 v 16 w(GET)p 626 2308 V 17 w(ELEMENTS)f FC(can)g(also)g(b)q(e)g(used)h (after)e(a)h(prob)q(e)g(to)g(\014nd)h(the)f(n)o(um)o(b)q(er)75 2365 y(of)20 b(elemen)o(ts)h(in)g(the)f(prob)q(ed)h(message.)34 b(Note)20 b(that)f(the)h(t)o(w)o(o)f(functions)i Fv(MPI)p 1507 2365 V 16 w(GET)p 1611 2365 V 17 w(COUNT)g FC(and)75 2421 y Fv(MPI)p 160 2421 V 16 w(GET)p 264 2421 V 17 w(ELEMENTS)15 b FC(return)g(the)g(same)g(v)m(alues)i(when)e(they)h(are)e(used)i(with) g(basic)g(datat)o(yp)q(es.)166 2554 y Fn(Implemen)o(tati)o(on)i(note:) 58 b Fq(The)19 b(curren)o(t)i(de\014nition)e(implies)e(that)i(a)g (receiv)o(e)h(cannot)g(c)o(hange)f(the)75 2604 y(v)n(alue)d(of)g (storage)h(b)q(et)o(w)o(een)h(the)f(en)o(tries)g(de\014ned)h(to)e(comp) q(ose)g(the)h(comm)o(unication)c(bu\013er.)27 b(In)17 b(particular,)75 2654 y(the)c(de\014nition)g(implies)e(that)i(padding)f (space)i(in)e(a)h(structure)i(should)d(not)h(b)q(e)g(mo)q(di\014ed)f (when)h(suc)o(h)h(structure)75 2704 y(is)f(copied)g(from)f(one)h(pro)q (cess)i(to)e(another.)18 b(This)13 b(w)o(ould)f(prev)o(en)o(ts)j(the)f (ob)o(vious)e(optimization)e(of)j(cop)o(ying)f(the)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 58 66 58 65 bop 75 -100 a FC(58)613 b Fs(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fq(structure,)e (together)f(with)e(the)i(padding,)e(as)h(one)g(con)o(tiguous)f(blo)q(c) o(k.)18 b(The)12 b(implemen)o(tatio)o(n)d(is)j(free)h(to)e(do)h(this)75 95 y(optimization)h(when)j(it)f(do)q(es)i(not)e(impact)f(the)i(outcome) f(of)g(the)h(computation.)22 b(The)16 b(user)g(can)g(\\force")g(this)75 145 y(optimization)11 b(b)o(y)j(explicitly)f(including)g(padding)g(as)h (part)g(of)f(the)h(message.)75 351 y Fp(3.13.6)49 b(Examples)75 437 y FC(The)15 b(follo)o(wing)h(examples)g(illustrate)h(the)e(use)g (of)g(deriv)o(ed)h(datat)o(yp)q(es.)75 559 y Fv(First)f(example:)45 b FC(Send)16 b(and)g(receiv)o(e)g(a)e(section)i(of)f(a)g(3D)g(arra)o(y) l(.)218 667 y Fo(REAL)23 b(a\(100,100,100\),)f(e\(9,9,9\))218 724 y(INTEGER)h(oneslice,)g(twoslice,)f(threeslice,)h(sizeofreal,)f (myrank,)h(ierr)218 780 y(MPI_STATUS)g(status)75 893 y(C)143 b(extract)23 b(the)g(section)g(a\(1:17:2,)g(3:11,)g(2:10\))75 949 y(C)143 b(and)23 b(store)h(it)f(in)h(e\(*,*,*\).)218 1062 y(CALL)f(MPI_COMM_RANK\(MPI_ALL,)e(myrank\))218 1175 y(CALL)i(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 1288 y(C)119 b(create)23 b(datatype)g(for)g(a)h(1D)g(section)218 1345 y(CALL)f(MPI_TYPE_VECTOR\()f(9,)i(1,)f(2,)h(MPI_REAL,)f(oneslice,) f(ierr\))75 1458 y(C)119 b(create)23 b(datatype)g(for)g(a)h(2D)g (section)218 1514 y(CALL)f(MPI_TYPE_HVECTOR\(9,)f(1,)h(100*sizeofreal,) f(oneslice,)h(twoslice,ierr\))75 1627 y(C)119 b(create)23 b(datatype)g(for)g(the)h(entire)f(section)218 1683 y(CALL)g (MPI_TYPE_HVECTOR\()f(9,)i(1,)f(100*100*sizeofreal,)e(twoslice,)i(1,) 767 1740 y(threeslice,)f(ierr\))218 1853 y(CALL)h(MPI_TYPE_COMMIT\()f (threeslice,)h(ierr\))218 1909 y(CALL)g(MPI_SENDRECV\(a\(1,3,2\),)e(1,) j(threeslice,)e(myrank,)h(0,)h(e,)f(9*9*9,)648 1966 y(MPI_REAL,)f (myrank,)h(0,)h(MPI_ALL,)f(status,)g(ierr\))75 2087 y Fv(Second)17 b(example:)44 b FC(Cop)o(y)15 b(the)g(\(strictly\))g(lo)o (w)o(er)g(triangular)g(part)g(of)g(a)g(matrix.)218 2195 y Fo(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 2252 y(INTEGER)47 b(disp\(100\),)22 b(blocklen\(100\),)g(ltype,)h(myrank,)g (ierr)218 2308 y(MPI_STATUS)g(status)75 2421 y(C)119 b(copy)23 b(lower)h(triangular)e(part)h(of)h(array)f(a)75 2478 y(C)119 b(onto)23 b(lower)h(triangular)e(part)h(of)h(array)f(b)218 2591 y(CALL)g(MPI_COMM_RANK\(MPI_ALL,)e(myrank\))75 2704 y(C)119 b(compute)23 b(start)g(and)h(size)f(of)h(each)f(column)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 59 67 59 66 bop 75 -100 a Fs(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FC(59)218 45 y Fo(DO)24 b(i=1,)f(100)266 102 y(disp\(i\))g(=)h (100*\(i-1\))e(+)i(i)266 158 y(block\(i\))f(=)g(100-i)218 214 y(END)h(DO)75 327 y(C)119 b(create)23 b(datatype)g(for)g(lower)h (triangular)e(part)218 384 y(CALL)h(MPI_TYPE_INDEX\()f(100,)i(block,)f (disp,)g(MPI_REAL,)f(ltype,)i(ierr\))218 497 y(CALL)f (MPI_TYPE_COMMIT\(ltype,)e(ierr\))218 553 y(CALL)i(MPI_SENDRECV\()g(a,) g(1,)h(ltype,)f(myrank,)g(0,)g(b,)h(1,)552 610 y(ltype,)f(myrank,)g(0,) h(MPI_ALL,)f(status,)f(ierr\))75 728 y Fv(Third)15 b(example:)45 b FC(T)l(ransp)q(ose)15 b(a)g(matrix)218 826 y Fo(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 882 y(INTEGER)g(row,)g(xpose,)g (sizeofreal,)g(myrank,)g(ierr)218 938 y(MPI_STATUS)g(status)75 1051 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 1164 y(CALL)g(MPI_COMM_RANK\(MPI_ALL,)e(myrank\))218 1277 y(CALL)i(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 1390 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 1447 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f(MPI_REAL,)g(row,)g (ierr\))75 1559 y(C)119 b(create)23 b(datatype)g(for)g(matrix)g(in)h (row-major)f(order)218 1616 y(CALL)g(MPI_TYPE_HVECTOR\()f(100,)h(1,)h (sizeofreal,)e(row,)i(xpose,)f(ierr\))218 1729 y(CALL)g (MPI_TYPE_COMMIT\()f(xpose,)h(ierr\))75 1842 y(C)119 b(send)23 b(matrix)h(in)f(row-major)g(order)g(and)g(receive)g(in)h (column)f(major)g(order)218 1898 y(CALL)g(MPI_SENDRECV\()g(a,)g(1,)h (xpose,)f(myrank,)g(0,)g(b,)h(100*100,)457 1955 y(MPI_REAL,)e(myrank,)h (0,)h(MPI_ALL,)f(status,)g(ierr\))166 2052 y FC(Another)15 b(approac)o(h)g(to)f(the)i(transp)q(ose)f(problem:)218 2139 y Fo(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 2195 y(INTEGER)47 b(disp\(2\),)23 b(blocklen\(2\),)f(type\(2\),)h(row,) g(row1,)g(sizeofreal)218 2252 y(INTEGER)47 b(myrank,)23 b(ierr)218 2308 y(MPI_STATUS)g(status)218 2421 y(CALL)g (MPI_COMM_RANK\(MPI_ALL,)e(myrank\))75 2534 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 2647 y(CALL)g(MPI_TYPE_EXTENT\()f (MPI_REAL,)h(sizeofreal,)f(ierr\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 60 68 60 67 bop 75 -100 a FC(60)613 b Fs(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fo(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 102 y(CALL)f (MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f(MPI_REAL,)g(row,)g(ierr\))75 214 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row,)f(with)h(the)f (extent)g(of)h(one)f(real)g(number)218 271 y(disp\(1\))g(=)h(0)218 327 y(disp\(2\))f(=)h(sizeofreal)218 384 y(type\(1\))47 b(=)24 b(row)218 440 y(type\(2\))47 b(=)24 b(MPI_UB)218 497 y(blocklen\(1\))46 b(=)24 b(1)218 553 y(blocklen\(2\))46 b(=)24 b(1)218 610 y(CALL)f(MPI_TYPE_STRUCT\()f(2,)i(blocklen,)e(disp,) i(type,)f(row1,)g(ierr\))218 723 y(CALL)g(MPI_TYPE_COMMIT\()f(row1,)h (ierr\))75 835 y(C)119 b(send)23 b(100)h(rows)f(and)h(receive)f(in)g (column)g(major)g(order)218 892 y(CALL)g(MPI_SENDRECV\()g(a,)g(100,)g (row1,)h(myrank,)f(0,)g(b,)h(100*100,)457 948 y(MPI_REAL,)e(myrank,)h (0,)h(MPI_ALL,)f(status,)g(ierr\))75 1071 y Fv(F)o(ourth)15 b(example:)45 b FC(manipulate)17 b(an)e(arra)o(y)f(of)g(structures.)75 1179 y Fo(struct)23 b(Partstruct)147 1236 y({)147 1292 y(int)95 b(class;)47 b(/*)23 b(particle)g(class)g(*/)147 1349 y(double)g(d[6];)71 b(/*)23 b(particle)g(coordinates)f(*/)147 1405 y(char)71 b(b[7];)g(/*)23 b(some)g(additional)g(information)f(*/) 147 1462 y(};)75 1574 y(struct)h(Partstruct)94 b(particle[1000];)75 1687 y(int)429 b(i,)24 b(dest,)f(rank;)75 1744 y(MPI_Comm)118 b(comm;)75 1913 y(/*)24 b(build)f(datatype)g(describing)f(structure)h (*/)75 2026 y(MPI_Datatype)f(Particletype;)75 2083 y(MPI_Datatype)g (type[3])h(=)h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 2139 y(int)238 b(blocklen[3])23 b(=)g({1,)h(6,)f(7};)75 2195 y(MPI_Aint)118 b(disp[3];)75 2365 y(/*)24 b(compute)e (displacements)h(of)g(structure)g(components)f(*/)75 2478 y(MPI_Address\()g(particle,)h(disp\);)75 2534 y(MPI_Address\()f (particle[0].d,)g(disp+1\);)75 2591 y(MPI_Address\()g(particle[0].b,)g (disp+2\);)75 2647 y(for)h(\(i=0;)h(i)f(<3;)h(i++\))f(disp[i])g(-=)h (disp[0];)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 61 69 61 68 bop 75 -100 a Fs(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FC(61)75 45 y Fo(MPI_Type_struct\()22 b(3,)h(blocklen,)g(disp,)g (type,)g(&Particletype\);)147 158 y(/*)g(If)h(compiler)f(does)g (padding)g(in)g(mysterious)g(ways,)147 214 y(the)g(following)g(may)g (be)h(safer)f(*/)75 327 y(MPI_Datatype)f(type1[4])h(=)h({MPI_INT,)e (MPI_DOUBLE,)h(MPI_CHAR,)f(MPI_UB};)75 384 y(int)238 b(blocklen1[4])22 b(=)i({1,)g(6,)f(7,)h(1};)75 440 y(MPI_Aint)118 b(disp1[4];)75 553 y(/*)24 b(compute)e(displacements)h(of)g(structure)g (components)f(*/)75 666 y(MPI_Address\()g(particle,)h(disp1\);)75 723 y(MPI_Address\()f(particle[0].d,)g(disp1+1\);)75 779 y(MPI_Address\()g(particle[0].b,)g(disp1+2\);)75 835 y(MPI_Address\()g(particle+1,)h(disp1+3\);)75 892 y(for)g(\(i=0;)h(i)f(<4;)h(i++\))f(disp1[i])g(-=)g(disp1[0];)75 1005 y(/*)h(build)f(datatype)g(describing)f(structure)h(*/)75 1118 y(MPI_Type_struct\()f(4,)h(blocklen1,)g(disp1,)g(type1,)g (&Particletype\);)409 1287 y(/*)h(4.1:)266 1344 y(send)f(the)h(entire)f (array)g(*/)75 1456 y(MPI_Commit\()f(Particletype\);)75 1513 y(MPI_Send\()h(particle,)f(1000,)h(Particletype,)g(dest,)g(tag,)g (comm\);)409 1682 y(/*)h(4.2:)266 1739 y(send)f(only)h(the)f(entries)g (of)g(class)h(zero)f(particles,)266 1795 y(preceded)g(by)g(the)h (number)f(of)g(such)h(entries)f(*/)75 1908 y(MPI_Datatype)f (Zparticles;)70 b(/*)24 b(datatype)f(describing)f(all)i(particles)791 1965 y(with)f(class)g(zero)h(\(needs)f(to)g(be)h(recomputed)791 2021 y(if)f(classes)g(change\))g(*/)75 2077 y(MPI_Datatype)f(Ztype;)75 2190 y(MPI_Aint)118 b(zdisp[1000];)75 2247 y(int)23 b(zblock[1000],)g (j,)g(k;)75 2303 y(int)g(zzblock[2])g(=)h({1,1};)75 2360 y(MPI_Aint)118 b(zzdisp[2];)75 2416 y(MPI_Datatype)22 b(zztype[2];)75 2529 y(/*)i(compute)e(displacements)h(of)g(class)g (zero)h(particles)e(*/)75 2586 y(j)i(=)f(0;)75 2642 y(for\(i=0;)g(i)g (<)h(1000;)f(i++\))123 2698 y(if)g(\(particle[i].class==0\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 62 70 62 69 bop 75 -100 a FC(62)613 b Fs(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)194 45 y Fo({)194 102 y(zdisp[j])23 b(=)h(i;)194 158 y(zblock[j])f(=)h(1;)194 214 y(j++;)194 271 y(})75 384 y(/*)g(create)f(datatype)f(for)i(class)f (zero)g(particles)47 b(*/)75 440 y(MPI_Type_indexed\()22 b(j,)h(zblock,)g(zdisp,)g(Particletype,)f(&Zparticles\);)75 553 y(/*)i(prepend)e(particle)h(count)g(*/)75 610 y(MPI_Address\(&j,)f (zzdisp\);)75 666 y(MPI_Address\(particle,)f(zzdisp+1\);)75 723 y(zztype[0])i(=)g(MPI_INT;)75 779 y(zztype[1])g(=)g(Zparticles;)75 835 y(MPI_Type_struct\(2,)e(zzblock,)i(zzdisp,)g(zztype,)g(&Ztype\);)75 948 y(MPI_Type_commit\()f(Ztype\);)75 1005 y(MPI_Send\()h(MPI_BOTTOM,)f (1,)i(Ztype,)f(dest,)g(tag,)g(comm\);)242 1174 y(/*)h(A)f(probably)g (more)g(efficient)g(way)h(of)f(defining)g(Zparticles)f(*/)75 1287 y(/*)i(consecutive)e(particles)h(with)g(index)g(zero)g(are)h (handled)f(as)g(one)h(block)f(*/)75 1344 y(j=0;)75 1400 y(for)g(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 1456 y(if)g (\(particle[i].index==0\))170 1513 y({)170 1569 y(for)h(\(k=i+1;)f(\(k) g(<)h(1000\)&&\(particle[k].index)d(==)i(0\))h(;)f(k++\);)170 1626 y(zdisp[j])g(=)h(i;)170 1682 y(zblock[j])f(=)h(k-i;)170 1739 y(j++;)170 1795 y(i)g(=)g(k;)170 1852 y(})75 1908 y(MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g(Particletype,)f (&Zparticles\);)457 2077 y(/*)h(4.3:)314 2134 y(send)g(the)g(first)h (two)f(coordinates)f(of)i(all)f(entries)g(*/)75 2247 y(MPI_Datatype)f(Allpairs;)142 b(/*)24 b(datatype)f(for)g(all)g(pairs)h (of)f(coordinates)g(*/)75 2360 y(MPI_Aint)g(sizeofentry;)75 2473 y(MPI_Type_extent\()f(Particletype,)g(sizeofentry\);)194 2586 y(/*)i(sizeofentry)e(can)i(also)f(be)h(computed)e(by)i (subtracting)e(the)i(address)266 2642 y(of)f(particle[0])g(from)g(the)h (address)f(of)g(particle[1])f(*/)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 63 71 63 70 bop 75 -100 a Fs(3.13.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b FC(63)75 45 y Fo(MPI_Type_hvector\()22 b(1000,)h(2,)g (sizeofentry,)g(MPI_REAL,)f(&Allpairs\);)75 102 y(MPI_Type_commit\()g (Allpairs\);)75 158 y(MPI_Send\()h(particle.d,)f(1,)i(Allpairs,)e (dest,)h(tag,)h(comm\);)218 271 y(/*)g(an)f(alternative)g(solution)f (to)i(4.3)g(*/)75 384 y(MPI_Datatype)e(Onepair;)71 b(/*)23 b(datatype)g(for)h(one)f(pair)g(of)h(coordinates,)e(with)695 440 y(the)i(extent)f(of)h(one)f(particle)g(entry)g(*/)75 497 y(MPI_Aint)g(disp2[3];)75 553 y(MPI_Datatype)f(type2[3])h(=)h ({MPI_LB,)e(MPI_DOUBLE,)h(MPI_UB};)75 610 y(int)g(blocklen2[3])g(=)g ({1,)h(2,)f(1};)75 723 y(MPI_Address\()f(particle,)h(disp2\);)75 779 y(MPI_Address\()f(particle[0].d,)g(disp2+1\);)75 835 y(MPI_Address\()g(particle+1,)h(disp2+2\);)75 892 y(for)g(\(i=0;)h(i<2;)f(i++\))g(disp2[i])g(-=)h(disp2[0];)75 1005 y(MPI_Type_struct\()e(3,)h(blocklen2,)g(disp2,)g(type2,)g (&Onepair\);)75 1061 y(MPI_Commit\()f(Onepair\);)75 1118 y(MPI_Send\()h(particle[0].d,)f(1000,)h(Onepair,)g(dest,)g(tag,)g (comm\);)75 1297 y Fv(Fifth)18 b(example:)45 b FC(same)17 b(arra)o(y)g(manipulations)i(as)f(in)g(the)g(previous)g(example,)h(but) f(using)h(absolute)75 1353 y(addresses)c(in)h(datat)o(yp)q(es.)75 1462 y Fo(struct)23 b(Partstruct)147 1518 y({)147 1574 y(int)g(class;)147 1631 y(double)g(d[6];)147 1687 y(char)g(b[7];)147 1744 y(};)75 1857 y(struct)g(Partstruct)g(particle[1000];)337 1970 y(/*)h(build)f(datatype)g(describing)g(first)g(array)g(entry)g(*/) 75 2083 y(MPI_Datatype)f(Particletype;)75 2139 y(MPI_Datatype)g (type[3])h(=)h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 2195 y(int)238 b(block[3])23 b(=)h({1,)f(6,)h(7};)75 2252 y(MPI_Aint)118 b(disp[3];)75 2365 y(MPI_Address\()22 b(particle,)h(disp\);)75 2421 y(MPI_Address\()f(particle[0].d,)g (disp+1\);)75 2478 y(MPI_Address\()g(particle[0].b,)g(disp+2\);)75 2534 y(MPI_Type_struct\()g(3,)h(block,)g(disp,)g(type,)h (&Particletype\);)75 2647 y(/*)g(Particletype)e(describes)g(first)i (array)f(entry)g(--)h(using)f(absolute)147 2704 y(addresses)f(*/)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 64 72 64 71 bop 75 -100 a FC(64)613 b Fs(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)505 102 y Fo(/*)23 b(5.1:)361 158 y(send)h(the)f(entire)g(array)g(*/)75 271 y(MPI_Commit\()f(Particletype\);)75 327 y(MPI_Send\()h(MPI_BOTTOM,) f(1000,)h(Particletype,)f(dest,)h(tag,)h(comm\);)481 497 y(/*)f(5.2:)290 553 y(send)g(the)h(entries)e(of)i(class)f(zero,)290 610 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)e(*/)75 723 y(MPI_Datatype)g(Zparticles,)h(Ztype;)75 835 y(MPI_Aint)g (zdisp[1000])75 892 y(int)g(zblock[1000],)g(i,)g(j,)h(k;)75 948 y(int)f(zzblock[2])g(=)h({1,1};)75 1005 y(MPI_Datatype)e (zztype[2];)75 1061 y(MPI_Aint)118 b(zzdisp[2];)75 1174 y(j=0;)75 1231 y(for)23 b(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 1287 y(if)g(\(particle[i].index==0\))170 1344 y({)170 1400 y(for)h(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d(==)i (0\))h(;)f(k++\);)170 1456 y(zdisp[j])g(=)h(i;)170 1513 y(zblock[j])f(=)h(k-i;)170 1569 y(j++;)170 1626 y(i)g(=)g(k;)170 1682 y(})75 1739 y(MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g (Particletype,)f(&Zparticles\);)75 1795 y(/*)i(Zparticles)e(describe)h (particles)f(with)i(class)f(zero,)g(using)147 1852 y(their)g(absolute)g (addresses*/)75 1965 y(/*)h(prepend)e(particle)h(count)g(*/)75 2021 y(MPI_Address\(&j,)f(zzdisp\);)75 2077 y(zzdisp[1])h(=)g (MPI_BOTTOM;)75 2134 y(zztype[0])g(=)g(MPI_INT;)75 2190 y(zztype[1])g(=)g(Zparticles;)75 2247 y(MPI_Type_struct\(2,)e(zzblock,) i(zzdisp,)g(zztype,)g(&Ztype\);)75 2360 y(MPI_Type_commit\()f(Ztype\);) 75 2416 y(MPI_Send\()h(MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g (comm\);)75 2595 y Fv(Sixth)16 b(example:)45 b FC(handling)17 b(of)e(unions.)75 2704 y Fo(union)23 b({)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 65 73 65 72 bop 75 -100 a Fs(3.13.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b FC(65)147 45 y Fo(int)118 b(ival;)147 102 y(float)70 b(fval;)218 158 y(})24 b(u[1000])75 271 y(int)f(utype;)75 384 y(/*)h(All)f(entries)g(of)h(u)f(have)h(identical)e(type;)h (variable)147 440 y(utype)g(keeps)g(track)g(of)h(their)f(current)g (type)g(*/)75 553 y(MPI_Datatype)70 b(type[2];)75 610 y(int)286 b(blocklen[2])22 b(=)i({1,1};)75 666 y(MPI_Aint)166 b(disp[2];)75 723 y(MPI_Datatype)70 b(mpi_utype[2];)75 779 y(MPI_Aint)166 b(i,j;)75 892 y(/*)24 b(compute)e(an)i(MPI)g (datatype)e(for)i(each)f(possible)g(union)g(type;)147 948 y(assume)g(values)g(are)g(left-aligned)f(in)i(union)f(storage.)g (*/)75 1061 y(MPI_Address\()f(u,)i(&i\);)75 1118 y(MPI_Address\()e (u+1,)i(&j\);)75 1174 y(disp[0])f(=)h(0;)f(disp[1])g(=)h(j-i)75 1231 y(type[1])f(=)h(MPI_UB;)75 1344 y(type[0])f(=)h(MPI_INT;)75 1400 y(MPI_Type_struct\(2,)d(blocklen,)i(disp,)g(type,)g (&mpi_utype[0]\);)75 1513 y(type[0])g(=)h(MPI_FLOAT;)75 1569 y(MPI_Type_struct\(2,)d(blocklen,)i(disp,)g(type,)g (&mpi_utype[1]\);)75 1682 y(for\(i=0;)g(i<2;)g(i++\))g (MPI_Type_commit\(mpi_utype[i]\))o(;)75 1795 y(/*)h(actual)f (communication)f(*/)75 1908 y(MPI_Send\(u,)g(1000,)i(mpi_utype[utype],) d(dest,)i(tag,)h(comm\);)75 2042 y Fp(3.13.7)49 b(Co)o(rrect)15 b(use)h(of)g(addresses)75 2132 y FC(Successiv)o(ely)23 b(declared)f(v)m(ariables)h(in)e(C)g(or)g(F)l(ortran)e(are)i(not)g (necessarily)h(stored)f(at)f(con)o(tiguous)75 2189 y(lo)q(cations.)i (Th)o(us,)16 b(care)g(m)o(ust)f(b)q(e)h(exercised)i(that)d(displacemen) o(ts)i(do)f(not)f(cross)h(from)f(one)h(v)m(ariable)75 2245 y(to)e(another.)20 b(Also,)15 b(in)g(mac)o(hines)h(with)f(segmen)o (ted)g(address)g(space,)g(addresses)g(are)g(not)f(unique)j(and)75 2302 y(address)11 b(arithmetic)h(has)g(some)f(p)q(eculiar)i(prop)q (erties.)19 b(Th)o(us,)12 b(use)g(of)f Fx(addresses)p FC(,)f(i.e.)20 b(displacemen)o(ts)75 2358 y(relativ)o(e)c(to)e(the)i (start)e(address)29 b Fl(MPI)p 733 2358 13 2 v 14 w(BOTTOM)p FC(,)14 b(has)h(to)g(b)q(e)h(restricted.)166 2417 y(V)l(ariables)k(b)q (elong)f(to)f(the)h(same)f Fx(sequen)o(tial)k(storage)d FC(if)g(they)g(b)q(elong)g(to)f(the)h(same)f(arra)o(y)l(,)75 2473 y(to)g(the)g(same)g Fo(COMMON)g FC(blo)q(c)o(k)h(in)g(F)l(ortran,) f(or)g(to)f(the)i(same)f(structure)g(in)h(C.)f(V)l(alid)i(addresses)e (are)75 2530 y(de\014ned)f(recursiv)o(ely)f(as)f(follo)o(ws:)131 2647 y(1.)22 b(The)g(function)g Fv(MPI)p 558 2647 14 2 v 16 w(ADDRESS)h FC(returns)e(a)h(v)m(alid)h(address,)h(when)e (passed)g(as)f(argumen)o(t)g(a)189 2704 y(v)m(ariable)16 b(of)f(the)g(calling)i(program.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 66 74 66 73 bop 75 -100 a FC(66)608 b Fs(CHAPTER)15 b(3.)35 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)131 45 y FC(2.)22 b(The)d Fv(buf)h FC(argumen)o(t)f(of)g(a)g(comm)o(unication)h(function) g(ev)m(aluates)h(to)d(a)h(v)m(alid)i(address,)f(when)189 102 y(passed)15 b(as)g(argumen)o(t)f(a)h(v)m(ariable)i(of)e(the)g (calling)i(program.)131 188 y(3.)22 b(If)15 b Fv(v)g FC(is)h(a)f(v)m(alid)i(address,)d(and)i Fv(i)f FC(is)g(an)h(in)o (teger,)f(then)g Fv(v+i)g FC(is)h(a)f(v)m(alid)i(address,)e(pro)o (vided)h Fv(v)f FC(and)189 245 y Fv(v+i)g FC(are)g(in)h(the)f(same)g (sequen)o(tial)h(storage.)131 331 y(4.)22 b(If)15 b Fv(v)g FC(is)h(a)f(v)m(alid)i(address)e(then)29 b Fl(MPI)p 820 331 13 2 v 15 w(BOTTOM)13 b(+)h(v)h FC(is)h(a)f(v)m(alid)i(address.)166 413 y(A)22 b(correct)g(program)g(uses)g(only)h(v)m(alid)i(addresses)d (to)g(iden)o(tify)i(the)e(lo)q(cations)h(of)f(en)o(tries)h(in)75 469 y(comm)o(unication)16 b(bu\013ers.)k(F)l(urthermore,)14 b(if)h Fv(u)h FC(and)f Fv(v)g FC(are)g(t)o(w)o(o)f(v)m(alid)j (addresses,)e(then)g(the)g(\(in)o(teger\))75 526 y(di\013erence)j Fv(u)g(-)f(v)g FC(can)g(b)q(e)h(computed)g(only)f(if)h(b)q(oth)f Fv(u)g FC(and)h Fo(v)f FC(are)f(in)i(the)g(same)e(sequen)o(tial)j (storage;)75 582 y(no)c(other)g(arithmetic)h(op)q(erations)f(can)g(b)q (e)h(meaningfully)i(executed)e(on)f(addresses.)166 639 y(W)l(e)f(sa)o(y)g(that)g(a)g(datat)o(yp)q(e)g(is)h Fx(absolute)g FC(if)g(all)h(displacemen)o(ts)g(within)f(this)g(datat)o(yp)q(e)f(are)g (v)m(alid)75 695 y(\(absolute\))h(addresses;)g(it)g(is)h Fx(relativ)o(e)g FC(otherwise.)166 752 y(A)f(correct)g(program)f(ob)q (eys)i(the)f(follo)o(wing)h(constrain)o(ts)e(in)i(the)g(use)f(of)g (datat)o(yp)q(es:)143 833 y FA(\017)23 b FC(If)j(the)g Fv(oldt)o(yp)q(e)h FC(argumen)o(t)e(used)i(in)g Fv(MPI)p 976 833 14 2 v 15 w(TYPE)p 1108 833 V 17 w(CONTIGUOUS)p FC(,)h Fv(MPI)p 1536 833 V 15 w(TYPE)p 1668 833 V 17 w(VECTOR)p FC(,)189 890 y Fv(MPI)p 274 890 V 15 w(TYPE)p 406 890 V 17 w(HVECTOR)p FC(,)20 b Fv(MPI)p 749 890 V 16 w(TYPE)p 882 890 V 17 w(INDEX)p FC(,)f(or)f Fv(MPI)p 1207 890 V 16 w(TYPE)p 1340 890 V 17 w(HINDEX)h FC(is)h(absolute,)g (then)189 946 y(all)15 b(addresses)g(within)g(it)g(m)o(ust)f(b)q(e)h (to)f(v)m(ariables)i(con)o(tained)f(within)g(the)g(same)f(arra)o(y)f (or)h(struc-)189 1003 y(ture;)j(the)g(result)h(datat)o(yp)q(e)e(is)i (also)f(absolute,)h(and)f(all)h(computed)f(addresses)h(m)o(ust)e(also)h (fall)189 1059 y(within)f(the)f(same)g(arra)o(y)f(or)h(structure.)143 1146 y FA(\017)23 b FC(If)18 b(all)g(en)o(tries)h(of)e(the)h Fv(a)o(rra)o(y)p 684 1146 V 14 w(of)p 735 1146 V 16 w(t)o(yp)q(es)i FC(argumen)o(t)d(of)g Fv(MPI)p 1210 1146 V 16 w(TYPE)p 1343 1146 V 17 w(STRUCT)h FC(are)g(absolute)g(ad-)189 1202 y(dresses)12 b(\(computed)g(b)o(y)g Fv(MPI)p 707 1202 V 15 w(ADDRESS)p FC(\),)h(then)f(the)g(result)g(datat)o(yp)q(e)g (is)g(also)g(absolute.)19 b(Eac)o(h)189 1259 y(new)14 b(address)h(computed)g(from)f(an)g(old)h(address)g(m)o(ust)f(fall)h (within)h(the)f(same)f(arra)o(y)f(or)h(record)189 1315 y(as)h(the)i(old)f(address.)23 b(\(Addresses)16 b(in)h(the)f(old)h(t)o (yp)q(e)f(ma)o(y)f(not)h(b)q(e)h(all)g(within)g(the)f(same)g(arra)o(y) 189 1372 y(or)e(record.\))143 1458 y FA(\017)23 b FC(If)15 b(a)g(comm)o(unication)h(call)h(is)e(in)o(v)o(ok)o(ed)h(with)g(argumen) o(ts)e Fv(buf)i FC(and)31 b Fv(datat)o(yp)q(e)p FC(,)16 b(then)g(either)g Fv(buf)189 1515 y FC(=)25 b Fl(MPI)p 326 1515 13 2 v 15 w(BOTTOM)13 b FC(and)g Fv(datat)o(yp)q(e)i FC(is)f(a)f(handle)i(to)d(an)i(absolute)f(datat)o(yp)q(e,)g(or)g Fv(buf)h FC(is)g(set)f(to)g(the)189 1571 y(address)h(of)g(a)g(program)f (v)m(ariable,)j(and)e(all)h(displacemen)o(ts)h(in)30 b Fv(datat)o(yp)q(e)16 b FC(relativ)o(e)e(to)g(this)h(base)189 1628 y(yield)h(addresses)g(that)e(are)h(within)i(the)e(same)g(sequen)o (tial)h(storage)e(as)h(this)h(v)m(ariable.)166 1709 y(In)21 b(summary)l(,)g(the)f(t)o(yp)q(e)h(constructors)e Fv(MPI)p 981 1709 14 2 v 16 w(TYPE)p 1114 1709 V 17 w(CONTIGUOUS,)j(MPI)p 1536 1709 V 15 w(TYPE)p 1668 1709 V 17 w(VECTOR,)75 1766 y(MPI)p 160 1766 V 16 w(TYPE)p 293 1766 V 17 w(HVECTOR,)15 b(MPI)p 631 1766 V 16 w(TYPE)p 764 1766 V 17 w(INDEX)f FC(and)29 b Fv(MPI)p 1114 1766 V 16 w(TYPE)p 1247 1766 V 17 w(HINDEX)14 b FC(can)h(b)q(e)g(recursiv)o(ely)h(ap-)75 1822 y(plied)22 b(to)e(build)j(datat)o(yp)q(es)d(that)f(will)k(com)o (bine)e(v)m(ariables)h(that)e(b)q(elong)h(to)f(the)h(same)f(sequen)o (tial)75 1878 y(storage;)d(v)m(ariables)i(that)e(do)g(not)g(b)q(elong)i (to)d(the)i(same)f(sequen)o(tial)i(storage)d(can)i(b)q(e)g(com)o(bined) g(to-)75 1935 y(gether)d(using)h(one)f(application)i(of)e Fv(MPI)p 788 1935 V 16 w(TYPE)p 921 1935 V 16 w(STRUCT)p FC(.)166 2067 y Fn(Implemen)o(tati)o(on)d(note:)166 2117 y Fq(There)i(is)f(no)f(need)i(to)f(distinguish)f(\(absolute\))h (addresses)i(and)e(\(relativ)o(e\))g(displacemen)o(ts)f(on)h(a)f(mac)o (hine)75 2167 y(with)k(con)o(tiguous)h(address)g(space:)25 b Fl(MPI)p 742 2167 13 2 v 14 w(BOTTOM)16 b Fq(is)h(zero,)g(and)g(b)q (oth)f(addresses)j(and)d(displacemen)o(ts)h(are)75 2217 y(in)o(tegers.)h(On)12 b(mac)o(hines)e(where)j(the)f(distinction)f(is)g (required,)h(addresses)i(are)e(recognized)g(as)g(expressions)h(that)75 2267 y(in)o(v)o(olv)o(e)g Fl(MPI)p 292 2267 V 14 w(BOTTOM)p Fq(.)75 2489 y Ft(3.14)59 b(Universal)20 b(communication)d(functions)75 2591 y Fx(This)c(section)g(is)g(not)g(part)g(of)f(the)h(MPI)e (de\014nition.)20 b FC(Its)11 b(goal)g(is)g(to)f(indicate)j(ho)o(w)d (the)h(de\014nition)75 2647 y(or)i(implemen)o(tation)h(of)f(the)g(p)q (oin)o(t)h(to)f(p)q(oin)o(t)g(comm)o(unication)h(functions)g(can)g(b)q (e)f(reduced)i(to)d(a)h(small)75 2704 y(n)o(um)o(b)q(er)j(of)e (primitiv)o(es.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 67 75 67 74 bop 75 -100 a Fs(3.14.)34 b(UNIVERSAL)17 b(COMMUNICA)l(TION)f (FUNCTIONS)633 b FC(67)166 45 y(W)l(e)16 b(assume)g(that)f(the)i (function)g Fv(MPI)p 846 45 14 2 v 15 w(TYPE)p 978 45 V 17 w(COMMIT)f FC(is)g(not)g(c)o(hanging)h(the)f(represen)o(tation)75 102 y(of)f(a)g(datat)o(yp)q(e,)f(so)h(that)f(this)i(function)g(is)g (redundan)o(t.)166 159 y(Comm)o(unication)g(with)h(a)e Fv(count)j FC(argumen)o(t)d(that)g(is)i Fw(>)d FC(1)i(can)g(b)q(e)h (reduced)g(to)e(comm)o(unication)75 216 y(with)k Fv(count)g(=1)p FC(,)g(b)o(y)g(replicating)h(the)37 b Fv(datat)o(yp)q(e)20 b FC(argumen)o(t)d Fv(count)j FC(times.)30 b(Comm)o(unication)19 b(with)75 272 y(a)d Fv(a)g(buf)h FC(argumen)o(t)f(that)g(is)h(not)31 b Fl(MPI)p 755 272 13 2 v 14 w(BOTTOM)16 b FC(can)g(b)q(e)h(reduced)h (to)e(comm)o(unication)h(with)f Fv(buf)i FC(=)75 329 y Fl(MPI)p 152 329 V 14 w(BOTTOM)d FC(b)o(y)g(creating)h(a)e(suitable)j (displaced)g(new)f(datat)o(yp)q(e.)j(Th)o(us:)166 386 y Fv(MPI)p 251 386 14 2 v 16 w(SEND\(buf,)c(count,)i(datat)o(yp)q(e,)f (dest,)g(tag,)f(comm\))e FC(is)75 486 y Fo(Type[1])23 b(=)h(datatype)75 542 y(MPI_ADDRESS\()e(buf,)i(Disp[1]\);)75 598 y(Blocklen[1])e(=)i(count;)75 655 y(MPI_TYPE_STRUCT\()e(1,)h(Type,) g(Disp,)h(Blocklen,)e(Newtype\))75 711 y(MPI_SEND\()h(MPI_BOTTOM,)f(1,) i(Newtype,)e(dest,)i(tag,)f(comm\))166 810 y FC(The)18 b(same)g(construction)h(applies)g(to)f(all)h(other)f(comm)o(unication)g (functions)h(with)g(argumen)o(ts)75 866 y Fv(count)e FC(and)f Fv(datat)o(yp)q(e)p FC(.)24 b(W)l(e)15 b(shall)i(henceforth)g (restrict)e(ourselv)o(es)i(to)e(comm)o(unication)h(that)f(in)o(v)o(olv) o(es)75 923 y(only)h(one)f(elemen)o(t)h(with)g(an)f(absolute)g(address) g(\()p Fv(count)i(=1)p FC(,)d(and)i Fv(buf)g FC(=)29 b Fl(MPI)p 1465 923 13 2 v 15 w(BOTTOM)p FC(\).)166 980 y(It)10 b(is)h(con)o(v)o(enien)o(t)g(to)e(to)h(mak)o(e)f(explicit)k (the)d Fx(p)q(ersistence)g FC(attribute)g(of)g(comm)o(unication)h(ob)s (jects.)75 1037 y(A)h Fv(p)q(ersistent)i FC(comm)o(unication)f(ob)s (ject)e(need)h(b)q(e)h(explicitly)h(deallo)q(cated)g(b)o(y)d(a)h Fv(MPI)p 1532 1037 14 2 v 16 w(REQUEST)p 1751 1037 V 17 w(FREE)75 1093 y FC(op)q(eration.)36 b(On)21 b(the)f(other)g(hand,)i (an)e Fv(ephemeral)h FC(comm)o(unication)g(ob)s(ject)f(is)g(go)q(o)q(d) h(for)f(a)g(single)75 1150 y(comm)o(unication.)g(It)14 b(is)g(deallo)q(cated)i(b)o(y)e(the)g(system)f(when)h(the)g(\014rst)g (comm)o(unication)h(it)f(is)g(used)h(for)75 1206 y(completes.)166 1264 y(The)g(function)h Fv(MPI)p 522 1264 V 16 w(COMM)p 680 1264 V 16 w(INIT)f FC(is)g(a)g(new)g(univ)o(ersal)i(function)f(for) e(the)h(creation)h(of)e(comm)o(u-)75 1320 y(nication)i(ob)s(jects:)75 1425 y Fv(MPI)p 160 1425 V 16 w(COMM)p 318 1425 V 16 w(INIT\(datat)o(yp)q(e,)g(source-dest,)h(tag,)d(comm,)f(op-mo)q(de,)i (p)q(ersistence,)j(request\))117 1503 y Fq(IN)155 b Fv(datat)o(yp)q(e) 424 b Fq(datat)o(yp)q(e)14 b(of)g(elemen)o(t)f(sen)o(t)i(or)f(receiv)o (ed)h(\(handle\))117 1581 y(IN)155 b Fv(source-dest)377 b Fq(rank)14 b(of)f(destination)h(or)g(source)h(\(in)o(teger\))117 1658 y(IN)155 b Fv(tag)526 b Fq(message)14 b(tag)f(\(in)o(teger\))117 1735 y(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 1813 y(IN)155 b Fv(op-mo)q(de)422 b Fq(one)64 b(of)127 b Fl(MPI)p 1269 1813 13 2 v 14 w(ST)m(AND)o(ARD)p Fq(,)f Fl(MPI)p 1715 1813 V 15 w(READ)o(Y)p Fq(,)905 1869 y Fl(MPI)p 982 1869 V 15 w(SYNCHRONOUS)12 b Fq(or)28 b Fl(MPI)p 1455 1869 V 14 w(RECV)117 1947 y Fq(IN)155 b Fv(p)q(ersistence)383 b Fq(one)14 b(of)27 b Fl(MPI)p 1119 1947 V 15 w(PERSISTENT)13 b Fq(or)28 b Fl(MPI)p 1534 1947 V 14 w(EPHEMERAL)117 2024 y Fq(OUT)108 b Fv(request)452 b Fq(comm)o(unication)11 b(request)k(\(handle\))75 2220 y Fp(3.14.1)49 b(P)o(ersistent)14 b(communication)j(objects)75 2308 y Fv(MPI)p 160 2308 14 2 v 16 w(CREA)l(TE)p 345 2308 V 17 w(SEND\(MPI)p 579 2308 V 16 w(BOTTOM,)f(1,)e(datat)o(yp)q(e,) j(dest,)f(tag,)f(comm,)e(request\))j FC(is)75 2421 y Fo(MPI_COMM_INIT\(datatype,)21 b(dest,)i(tag,)g(comm,)g(MPI_STANDARD,) 170 2478 y(MPI_PERSISTENT,)f(request\))166 2590 y FC(The)f(functions)g Fv(MPI)p 551 2590 V 16 w(CREA)l(TE)p 736 2590 V 17 w(RSEND)p FC(,)g Fv(MPI)p 1015 2590 V 16 w(CREA)l(TE)p 1200 2590 V 17 w(RSEND)g FC(and)g Fv(MPI)p 1560 2590 V 16 w(CREA)l(TE)p 1745 2590 V 17 w(RECV)75 2646 y FC(are)15 b(dealt)h(with)f(in)h(a)f (similar)h(manner.)166 2704 y(The)f(functions)h Fv(MPI)p 540 2704 V 16 w(ST)l(ART)g FC(and)g Fv(MPI)p 885 2704 V 15 w(REQUEST)p 1103 2704 V 18 w(FREE)g FC(are)f(primitiv)o(e.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 68 76 68 75 bop 75 -100 a FC(68)613 b Fs(CHAPTER)16 b(3.)29 b(POINT)16 b(TO)f(POINT)h(COMMUNICA)l(TION)75 45 y Fp(3.14.2)49 b(Nonblo)q(cking)19 b(communication)d(initiation)75 135 y Fv(MPI)p 160 135 14 2 v 16 w(ISEND\(MPI)p 406 135 V 15 w(BOTTOM,)g(1,)e(datat)o(yp)q(e,)j(dest,)f(tag,)f(comm,)e(request\)) j FC(is)75 254 y Fo(MPI_COMM_INIT\(datatype,)21 b(dest,)i(tag,)g(comm,) g(MPI_STANDARD,)194 310 y(MPI_EPHEMERAL,)f(request\))75 367 y(MPI_START\(request\))166 483 y FC(The)g(functions)g Fv(MPI)p 553 483 V 16 w(IRSEND)p FC(,)f Fv(MPI)p 844 483 V 16 w(ISSEND)h FC(and)g Fv(MPI)p 1214 483 V 16 w(IRECV)g FC(are)f(handled)i(in)g(a)e(similar)75 540 y(manner.)75 673 y Fp(3.14.3)49 b(Communication)17 b(completion)75 763 y FC(The)e(t)o(w)o(o)f(primitiv)o(e)j(completion)f(op)q(erations)g (are)e Fv(MPI)p 1063 763 V 16 w(W)l(AIT)l(ANY)i FC(and)g Fv(MPI)p 1467 763 V 16 w(TEST)l(ALL)p FC(.)166 822 y(The)c(function)h Fv(MPI)p 516 822 V 15 w(W)l(AIT)f FC(can)g(b)q(e)h(implemen)o(ted)g(b)o (y)f(a)f(call)i(to)e Fv(MPI)p 1368 822 V 16 w(W)l(AIT)l(ANY)i FC(with)f(an)24 b Fv(a)o(rra)o(y-)75 878 y(of-requests)17 b FC(argumen)o(t)d(of)h(length)h(one.)166 937 y(The)f(function)g Fv(MPI)p 521 937 V 16 w(W)l(AIT)l(ALL)g FC(can)g(b)q(e)g(implemen)o (ted)h(as)f(a)f(lo)q(op)h(where)g(an)g Fv(MPI)p 1621 937 V 16 w(W)l(AIT)g FC(call)g(is)75 993 y(executed)h(for)f(eac)o(h)g (successiv)o(e)h(request)g(in)g(the)f Fv(a)o(rra)o(y-of-requests)p FC(.)166 1052 y(The)f(function)g Fv(MPI)p 519 1052 V 16 w(TEST)g FC(can)f(b)q(e)h(implemen)o(ted)i(b)o(y)d(a)g(call)i(to)e Fv(MPI)p 1387 1052 V 16 w(TEST)l(ALL)g FC(with)h(an)g Fv(a)o(rra)o(y-)75 1108 y(of-requests)j FC(of)e(length)h(one.)166 1167 y(The)h(function)h Fv(MPI)p 526 1167 V 16 w(TEST)l(ANY)h FC(can)e(b)q(e)h(implemen)o(ted)h(as)d(a)h(lo)q(op)h(where)f(an)h Fv(MPI)p 1663 1167 V 15 w(TEST)g FC(call)75 1223 y(is)e(executed)g(for) e(eac)o(h)i(successiv)o(e)g(request)f(in)h(the)g Fv(a)o(rra)o (y-of-request)p FC(.)75 1357 y Fp(3.14.4)49 b(Blo)q(cking)18 b(communication)75 1447 y Fv(MPI)p 160 1447 V 16 w(SEND\(MPI)p 393 1447 V 16 w(BOTTOM,)d(1,)g(datat)o(yp)q(e,)h(dest,)h(tag,)d(comm\)) f FC(is)75 1565 y Fo(MPI_COMM_INIT\(datatype,)21 b(dest,)i(tag,)g (comm,)g(MPI_STANDARD,)218 1622 y(MPI_EPHEMERAL,)f(request\))75 1678 y(MPI_START\(request\))75 1735 y(MPI_WAIT\(request,)g (dontcarestatus\))166 1851 y FC(The)15 b(functions)h Fv(MPI)p 540 1851 V 16 w(RSEND)g FC(and)f Fv(MPI)p 888 1851 V 16 w(SSEND)h FC(are)f(handled)i(in)f(a)f(similar)h(manner.)166 1910 y Fv(MPI)p 251 1910 V 16 w(RECV\(MPI)p 482 1910 V 16 w(BOTTOM,)g(1,)e(datat)o(yp)q(e,)j(source,)e(tag,)g(comm\))e FC(is)75 2029 y Fo(MPI_COMM_INIT\(datatype,)21 b(dest,)i(tag,)g(comm,)g (MPI_RECV,)g(MPI_EPHEMERAL,)f(request\))75 2085 y(MPI_START\(request\)) 75 2142 y(MPI_WAIT\(request,)g(status\))75 2275 y Fp(3.14.5)49 b(Prob)q(e)17 b(and)f(cancel)75 2365 y FC(The)f(functions)h Fv(MPI)p 449 2365 V 16 w(PROBE)h FC(and)e Fv(MPI)p 800 2365 V 16 w(CANCEL)g FC(are)g(primitiv)o(e.)75 2499 y Fp(3.14.6)49 b(Return)16 b(status)75 2589 y FC(The)k(functions)g Fv(MPI)p 458 2589 V 16 w(GET)p 562 2589 V 16 w(SOURCE)p FC(,)h Fv(MPI)p 868 2589 V 16 w(GET)p 972 2589 V 16 w(T)l(A)o(G)p FC(,)f Fv(MPI)p 1189 2589 V 16 w(GET)p 1293 2589 V 16 w(LEN)p FC(,)f Fv(MPI)p 1507 2589 V 16 w(PROBE)p 1671 2589 V 18 w(LEN)p FC(,)g(and)75 2645 y Fv(MPI)p 160 2645 V 16 w(TEST)p 290 2645 V 16 w(CANCELLED)d FC(are)e(primitiv)o(e.)166 2704 y(These)h(functions)h(are)f(simple)i(macros)d(that)h(access)g (records)g(in)h(a)f(structure.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 69 77 69 76 bop 75 -100 a Fs(3.14.)34 b(UNIVERSAL)17 b(COMMUNICA)l(TION)f (FUNCTIONS)633 b FC(69)75 45 y Fp(3.14.7)49 b(send-receive)15 b(and)i(exchange)75 131 y Fv(MPI)p 160 131 14 2 v 16 w(SENDRECV\()11 b(send)p 520 131 V 18 w(buf,)g(send)p 705 131 V 18 w(count,)h(send)p 936 131 V 18 w(t)o(yp)q(e,)g(dest,)g (recv)p 1237 131 V 17 w(buf,)f(recv)p 1415 131 V 17 w(count,)h(recv)p 1639 131 V 16 w(t)o(yp)q(e,)g(source,)75 187 y(tag,)j(comm,)e(status\)) k FC(is)75 294 y Fo(MPI_ISEND\(send_buf,)k(send_count,)i(send_type,)f (dest,)h(tag,)h(comm,)f(request[0]\))75 350 y(MPI_IRECV\(recv_buf,)e (recv_count,)i(recv_type,)f(source,)h(tag,)g(comm,)h(request[1]\))75 407 y(MPI_WAITALL\(2,)e(request,)h(status_array\))75 463 y(status)g(=)h(status_array[1])166 569 y FC(The)13 b(non)o(blo)q(c)o(king)h(sends)f(and)g(receiv)o(es)h(can)f(b)q(e)g (replaced)h(b)o(y)f(their)g(primitiv)o(e)h(implemen)o(tation.)166 626 y(The)j(function)g Fv(MPI)p 525 626 V 16 w(SENDRECV)p 773 626 V 18 w(REPLA)o(CE)g FC(can)g(b)q(e)g(handled)h(in)g(a)e (similar)i(manner;)f(a)f(tem-)75 682 y(p)q(orary)j(bu\013er)h(need)g(b) q(e)h(allo)q(cated)f(to)f(replicate)i(the)f(send)g(and)g(receiv)o(e)h (bu\013er.)33 b(A)20 b(size)g(for)g(this)75 739 y(bu\013er)15 b(can)h(b)q(e)f(computed)h(using)g(the)f(function)h Fv(MPI)p 1020 739 V 16 w(TYPE)p 1153 739 V 17 w(SIZE)p FC(.)75 860 y Fp(3.14.8)49 b(Derived)16 b(datat)o(yp)q(es)75 946 y FC(W)l(e)22 b(ha)o(v)o(e)g(outlined)h(in)g(section)g(3.13.1)d(ho) o(w)i(eac)o(h)g(datat)o(yp)q(e)f(constructor)g(can)i(b)q(e)f(expressed) h(in)75 1003 y(terms)h(of)g(the)g(next)h(one.)47 b(Th)o(us)25 b(all)g(datat)o(yp)q(e)f(constructors)g(can)g(b)q(e)h(expressed)g(in)h (terms)d(of)75 1059 y(the)12 b(constructor)f Fv(MPI)p 472 1059 V 16 w(TYPE)p 605 1059 V 16 w(STRUCT)p FC(;)h(the)g(functions) h Fv(MPI)p 1172 1059 V 15 w(TYPE)p 1304 1059 V 17 w(FREE)p FC(,)f Fv(MPI)p 1537 1059 V 16 w(TYPE)p 1670 1059 V 17 w(EXTENT)p FC(,)75 1116 y Fv(MPI)p 160 1116 V 16 w(TYPE)p 293 1116 V 17 w(SIZE)p FC(,)i Fv(MPI)p 512 1116 V 16 w(TYPE)p 645 1116 V 17 w(COUNT)i FC(and)f Fv(MPI)p 1004 1116 V 16 w(ADDRESS)h FC(are)f(primitiv)o(e.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 70 78 70 77 bop 75 356 a Fu(Chapter)34 b(4)75 564 y Fy(Collecti)q(v)m(e)42 b(Comm)m(unication)75 805 y Ft(4.1)59 b(Intro)r(duction)75 906 y FC(Collectiv)o(e)13 b(comm)o(unication)f(is)g(de\014ned)h(to)e(b) q(e)h(comm)o(unication)g(that)f(in)o(v)o(olv)o(es)h(a)f(group)g(of)g (pro)q(cesses.)75 963 y(The)k(functions)h(pro)o(vided)g(b)o(y)g(the)f (MPI)g(collectiv)o(e)i(comm)o(unication)f(include:)143 1055 y FA(\017)23 b FC(Broadcast)14 b(from)g(one)i(mem)o(b)q(er)f(to)g (all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 1148 y FA(\017)23 b FC(Barrier)15 b(sync)o(hronization)h(across)f(all)h(group)f(mem)o(b)q (ers)143 1242 y FA(\017)23 b FC(Gather)14 b(data)h(from)f(all)i(group)f (mem)o(b)q(ers)g(to)g(one)g(mem)o(b)q(er.)143 1335 y FA(\017)23 b FC(Scatter)14 b(data)h(from)f(one)i(mem)o(b)q(er)f(to)g (all)h(mem)o(b)q(ers)f(of)g(a)g(group.)143 1428 y FA(\017)23 b FC(Global)15 b(op)q(erations)g(suc)o(h)h(as)e(sum,)h(max,)f(min,)i (etc.,)e(w)o(ere)h(the)g(result)g(is)h(kno)o(wn)e(b)o(y)h(all)h(group) 189 1485 y(mem)o(b)q(ers)e(and)h(a)g(v)m(ariation)g(where)g(the)g (result)g(is)g(kno)o(wn)f(b)o(y)h(only)g(one)g(mem)o(b)q(er.)20 b(The)15 b(abilit)o(y)189 1541 y(to)f(ha)o(v)o(e)h(user)g(de\014ned)i (global)f(op)q(erations.)143 1634 y FA(\017)23 b FC(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g(called)h (parallel)h(pre\014x\).)143 1728 y FA(\017)23 b FC(Broadcast)14 b(from)g(all)j(mem)o(b)q(ers)e(to)f(all)j(mem)o(b)q(ers)e(of)g(a)g (group.)143 1821 y FA(\017)23 b FC(Scatter/Gather)10 b(data)h(from)h(all)g(mem)o(b)q(ers)g(to)g(all)g(mem)o(b)q(ers)g(of)g (a)g(group)f(\(also)h(called)h(complete)189 1877 y(exc)o(hange)i(or)g (all-to-all\).)75 1970 y(While)23 b(v)o(endors)f(ma)o(y)f(optimize)i (some)e(of)h(these)g(collectiv)o(e)h(routines)f(for)g(their)g(arc)o (hitectures,)h(a)75 2026 y(complete)15 b(library)g(of)f(the)h (collectiv)o(e)h(comm)o(unication)f(routines)g(can)g(b)q(e)g(written)f (en)o(tirely)i(using)f(the)75 2083 y(MPI)g(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication)g(functions)g(and)f(a)g(few)g(auxiliary)i(functions.) 166 2139 y(A)h(collectiv)o(e)i(op)q(eration)e(is)g(executed)h(b)o(y)f (ha)o(ving)g(all)h(pro)q(cesses)f(in)h(the)f(group)g(call)h(the)f(com-) 75 2195 y(m)o(unication)f(routine,)f(with)g(matc)o(hing)g(argumen)o (ts.)21 b(The)16 b(syn)o(tax)f(and)h(seman)o(tics)g(of)f(the)h (collectiv)o(e)75 2252 y(op)q(erations)e(are)g(de\014ned)i(to)e(b)q(e)g (consisten)o(t)h(with)f(the)h(syn)o(tax)e(and)i(seman)o(tics)f(of)g (the)g(p)q(oin)o(t-to-p)q(oin)o(t)75 2308 y(op)q(erations.)25 b(Th)o(us)17 b(general)g(datat)o(yp)q(es)f(are)h(allo)o(w)o(ed)g(and)g (m)o(ust)f(matc)o(h)h(b)q(et)o(w)o(een)g(sending)h(and)f(re-)75 2365 y(ceiving)j(pro)q(cesses)g(as)e(sp)q(eci\014ed)j(in)f(the)f(c)o (hapter)f(on)h(p)q(oin)o(t-to-p)q(oin)o(t)h(functions.)31 b(One)20 b(of)e(the)h(k)o(ey)75 2421 y(argumen)o(ts)12 b(is)h(a)g(comm)o(unicator)f(that)g(de\014nes)i(the)f(group)g(of)f (participating)i(pro)q(cesses)f(and)g(pro)o(vides)75 2478 y(a)j(con)o(text)g(for)g(the)g(op)q(eration.)24 b(Sev)o(eral)17 b(collectiv)o(e)h(routines)f(suc)o(h)f(as)g(broadcast)g (and)h(gather)e(ha)o(v)o(e)75 2534 y(a)g(single)i(originating)f(or)f (receiving)i(pro)q(cess.)k(In)16 b(this)g(c)o(hapter)f(suc)o(h)h(pro)q (cesses)g(are)f(called)i(the)e Fr(r)n(o)n(ot)p FC(.)75 2591 y(Some)e(argumen)o(ts)f(in)h(the)g(collectiv)o(e)i(functions)e (are)g(sp)q(eci\014ed)h(as)f(\\signi\014can)o(t)g(only)h(at)e(ro)q (ot".)18 b(These)75 2647 y(argumen)o(ts)12 b(are)h(ignored)g(for)g(all) h(participan)o(ts)f(except)g(the)h(ro)q(ot,)e(and)h(can)g(b)q(e)h(set)e (to)h(an)o(y)f(v)m(alue.)21 b(The)75 2704 y(reader)d(is)g(referred)g (to)f(c)o(hapter)g(3)g(for)h(information)f(concerning)i(comm)o (unication)g(bu\013ers,)e(general)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 71 79 71 78 bop 75 -100 a Fs(4.2.)34 b(COMMUNICA)l(TION)16 b(FUNCTIONS)942 b FC(71)75 45 y(datat)o(yp)q(es)11 b(and)g(t)o(yp)q(e)h (matc)o(hing)f(rules;)i(and)f(to)e(c)o(hapter)i(5)f(for)f(information)i (on)f(ho)o(w)g(to)g(de\014ne)h(groups)75 102 y(and)j(create)g(comm)o (unicators.)166 171 y(Collectiv)o(e)i(routines)g(can)f(\(but)g(are)g (not)g(required)h(to\))e(return)i(as)e(so)q(on)h(as)g(their)h (participation)75 227 y(in)i(the)g(collectiv)o(e)h(comm)o(unication)f (is)g(complete.)31 b(The)19 b(completion)g(of)f(a)h(call)g(indicates)h (that)e(the)75 284 y(caller)f(is)f(no)o(w)f(free)h(to)f(access)g(the)h (lo)q(cations)g(in)h(the)e(comm)o(unication)i(bu\013er.)k(It)15 b(do)q(es)h(not)f(indicate)75 340 y(that)k(other)f(pro)q(cesses)i(in)g (the)g(group)e(ha)o(v)o(e)h(started)g(the)g(op)q(eration)h(\(unless)g (otherwise)f(indicated)75 397 y(in)h(the)f(description)h(of)f(the)g(op) q(eration\).)31 b(The)19 b(successful)h(completion)g(of)f(a)g (collectiv)o(e)h(comm)o(uni-)75 453 y(cation)e(call)g(ma)o(y)f(dep)q (end)i(on)e(the)h(execution)g(of)f(a)g(matc)o(hing)h(call)g(at)f(all)h (pro)q(cesses)g(in)g(the)g(group.)75 510 y(Th)o(us,)i(a)f(collectiv)o (e)i(comm)o(unication)f(call)h(ma)o(y)l(,)e(or)g(ma)o(y)f(not,)i(ha)o (v)o(e)f(the)g(e\013ect)h(of)e(sync)o(hronizing)75 566 y(all)g(calling)h(pro)q(cesses.)26 b(Collectiv)o(e)19 b(comm)o(unication)f(calls)g(ma)o(y)f(use)g(the)h(same)f(comm)o (unicators)f(as)75 623 y(p)q(oin)o(t)i(to)f(p)q(oin)o(t)g(comm)o (unication;)i(MPI)e(guaran)o(tees)g(that)f(messages)h(generated)h(on)f (b)q(ehalf)h(of)f(col-)75 679 y(lectiv)o(e)e(comm)o(unication)g(calls)f (will)i(not)d(b)q(e)i(confused)f(with)g(messages)g(generated)f(b)o(y)h (p)q(oin)o(t)g(to)f(p)q(oin)o(t)75 735 y(comm)o(unication.)166 805 y(A)h(more)g(detailed)i(discussion)g(of)e(the)g(correct)g(use)h(of) e(the)i(collectiv)o(e)h(routines)f(can)f(b)q(e)h(found)g(at)75 861 y(the)g(end)h(of)f(this)h(c)o(hapter.)75 1078 y Ft(4.2)59 b(Communication)18 b(F)n(unctions)75 1205 y FC(The)d(k)o(ey)h(concept)f (of)g(the)g(collectiv)o(e)i(functions)f(is)g(to)f(ha)o(v)o(e)f(a)h (\\group")g(of)g(participating)h(pro)q(cesses.)75 1261 y(The)d(routines)f(do)h(not)e(ha)o(v)o(e)h(a)g(group)g(iden)o(ti\014er) i(as)e(an)g(explicit)j(argumen)o(t.)j(Instead,)13 b(there)f(is)h(a)f (com-)75 1318 y(m)o(unicator)g(argumen)o(t.)18 b(In)12 b(this)g(c)o(hapter)g(a)f(comm)o(unicator)g(can)h(b)q(e)h(though)o(t)e (of)g(as)g(a)h(group)f(iden)o(ti\014er)75 1374 y(link)o(ed)20 b(with)e(a)g(con)o(text.)27 b(\(In)o(ter-comm)o(unicators,)18 b(that)f(is)i(b)q(et)o(w)o(een)f(groups)f(comm)o(unicators,)h(are)75 1430 y(not)d(allo)o(w)o(ed)g(in)h(the)g(collectiv)o(e)h(functions.\))75 1647 y Ft(4.3)59 b(Ba)n(rrier)21 b(synchronization)75 1821 y Fv(MPI)p 160 1821 14 2 v 16 w(BARRIER\()16 b(comm)d(\))117 1911 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))75 2049 y Fo(int)23 b(MPI)p 245 2049 15 2 v 17 w(Barrier\(MPI)p 526 2049 V 16 w(Comm)g(comm)h(\))75 2148 y(MPI)p 150 2148 V 17 w(BARRIER\(COMM,)e(IERROR\))170 2204 y(INTEGER)h(COMM,)h(IERROR)166 2304 y Fv(MPI)p 251 2304 14 2 v 16 w(BARRIER)15 b FC(blo)q(c)o(ks)g(the)f(caller)h(un)o (til)g(all)f(group)g(mem)o(b)q(ers)g(ha)o(v)o(e)f(called)j(it;)e(the)g (call)h(returns)75 2360 y(at)g(an)o(y)f(pro)q(cess)i(only)g(after)e (all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered)h(the)f(call.)75 2577 y Ft(4.4)59 b(Data)19 b(move)g(functions)75 2704 y FC(Figure)c(4.1)g(illustrates)h(the)f(the)g(di\013eren)o(t)h (collectiv)o(e)h(mo)o(v)o(e)d(functions)i(supp)q(orted)g(b)o(y)f(MPI.) -32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 72 80 72 79 bop 75 -100 a FC(72)703 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 2420 y @beginspecial @setspecial %%BeginDocument: coll-fig1.ps /arrowdict 13 dict def % Local storage for the procedure % ``arrow.'' /arrow % The procedure ``arrow'' adds an { arrowdict begin % arrow shape to the current path. /headlength exch def % It takes seven arguments: the x /halfheadthickness exch 2 div def % and y coordinates of the tail /halfthickness exch 2 div def % (imagine that a line has been /tipy exch def /tipx exch def % drawn down the center of the /taily exch def /tailx exch def % arrow from the tip to the tail, % then x and y lie on this line), % the x and y coordinates of the % tip of the arrow, the thickness % of the arrow in the tail % portion, the thickness of the % arrow at the widest part of the % arrowhead and the length of the % arrowhead. /dx tipx tailx sub def % Compute the differences in x and /dy tipy taily sub def % y for the tip and tail. These /arrowlength dx dx mul dy dy mul add % will be used to compute the sqrt def % length of the arrow and to /angle dy dx atan def % compute the angle of direction % that the arrow is facing with % respect to the current user % coordinate system origin. /base arrowlength headlength sub def % Compute where the base of the % arrowhead will be. /savematrix matrix currentmatrix def % Save the current user coordinate % system. We are using the same % strategy to localize the effect % of transformations as was used % in the program to draw an % ellipse. tailx taily translate % Translate to the starting point % of the tail. angle rotate % Rotate the x-axis to correspond % with the center line of the % arrow. 0 halfthickness neg moveto % Add the arrow shape to the % current path. base halfthickness neg lineto base halfheadthickness neg lineto arrowlength 0 lineto base halfheadthickness lineto base halfthickness lineto 0 halfthickness lineto closepath savematrix setmatrix % Restore the current user % coordinate system. end } def /Box { /height exch def /length exch def length 0 rlineto 0 height rlineto length neg 0 rlineto closepath } def /Gdict 200 dict def /Grid { Gdict begin /ny exch def /nx exch def /dely exch def /delx exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def 0 1 ny { pop x y moveto lenx 0 rlineto stroke /y y dely add def} for /y ypos def /x xpos def 0 1 nx { pop x y moveto 0 leny rlineto stroke /x x delx add def} for end } def /GLdict 300 dict def /GridLabels { GLdict begin /shift exch def /raise exch def /yoff exch def /xoff exch def /p1 exch def /p2 exch def /ny exch def /nx exch def /dely exch def /delx exch def /Darray exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def /dx3 delx 3 div def /dy3 dely 3 div def /ix -1 def /iy ny 1 sub def Darray{ aload pop /Subc exch def /Text exch def /ix ix 1 add def ix nx ge { /ix 0 def /iy iy 1 sub def} if /x xpos delx ix 0.5 add mul add /Helvetica findfont p1 scalefont setfont Text stringwidth pop /Helvetica findfont p2 scalefont setfont Subc stringwidth pop add xoff add 2 div sub shift add def /y ypos dely iy 0.5 add mul add raise add def x y moveto /Helvetica findfont p1 scalefont Text show xoff yoff rmoveto /Helvetica findfont p2 scalefont Subc show } forall end clear } def 2 setlinecap 6.5 72 mul 320 sub 2 div 0 translate 0 150 moveto 20 20 6 6 Grid 0 150 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 225 200 225 12 24 18 arrow stroke 200 195 140 195 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (one-all scatter) dup stringwidth pop 170 exch 2 div sub 242 moveto show (one-all gather) dup stringwidth pop 170 exch 2 div sub 170 moveto show 220 150 moveto 20 20 6 6 Grid 220 150 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(1)] [()()] [()()] [()()] [()()] [()()] [(A)(2)] [()()] [()()] [()()] [()()] [()()] [(A)(3)] [()()] [()()] [()()] [()()] [()()] [(A)(4)] [()()] [()()] [()()] [()()] [()()] [(A)(5)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 0 moveto 20 20 6 6 Grid 0 0 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [(B)(0)] [(B)(1)] [(B)(2)] [(B)(3)] [(B)(4)] [(B)(5)] [(C)(0)] [(C)(1)] [(C)(2)] [(C)(3)] [(C)(4)] [(C)(5)] [(D)(0)] [(D)(1)] [(D)(2)] [(D)(3)] [(D)(4)] [(D)(5)] [(E)(0)] [(E)(1)] [(E)(2)] [(E)(3)] [(E)(4)] [(E)(5)] [(F)(0)] [(F)(1)] [(F)(2)] [(F)(3)] [(F)(4)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 220 0 moveto 20 20 6 6 Grid 220 0 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(1)] [(B)(1)] [(C)(1)] [(D)(1)] [(E)(1)] [(F)(1)] [(A)(2)] [(B)(2)] [(C)(2)] [(D)(2)] [(E)(2)] [(F)(2)] [(A)(3)] [(B)(3)] [(C)(3)] [(D)(3)] [(E)(3)] [(F)(3)] [(A)(4)] [(B)(4)] [(C)(4)] [(D)(4)] [(E)(4)] [(F)(4)] [(A)(5)] [(B)(5)] [(C)(5)] [(D)(5)] [(E)(5)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 60 200 60 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (all-all scatter) dup stringwidth pop 170 exch 2 div sub 77 moveto show 0 300 moveto 20 20 6 6 Grid 0 300 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(B)(0)] [()()] [()()] [()()] [()()] [()()] [(C)(0)] [()()] [()()] [()()] [()()] [()()] [(D)(0)] [()()] [()()] [()()] [()()] [()()] [(E)(0)] [()()] [()()] [()()] [()()] [()()] [(F)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 360 200 360 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (all gather) dup stringwidth pop 170 exch 2 div sub 377 moveto show 220 300 moveto 20 20 6 6 Grid 220 300 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 450 moveto 20 20 6 6 Grid 0 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 510 200 510 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont 0 575 moveto (data) show (data) stringwidth pop 4 add 575 4 add (data) stringwidth pop 33 add 575 4 add 1 4 5 arrow fill (one-all broadcast) dup stringwidth pop 170 exch 2 div sub 527 moveto show gsave 0 570 (processes) stringwidth pop sub translate 90 rotate 0 5 moveto (processes) show -4 8 -33 8 1 4 5 arrow fill grestore 220 450 moveto 20 20 6 6 Grid 220 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels showpage %%EndDocument @endspecial 98 x FC(Figure)f(4.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2574 y(eac)o(h)g(ro)o(w)g(of)g(b)q(o)o(xes)g (represen)o(ts)g(data)g(lo)q(cations)h(in)g(one)g(pro)q(cess.)k(Th)o (us,)15 b(in)h(the)f(one-all)i(broadcast,)75 2631 y(initially)k(just)e (the)f(\014rst)h(pro)q(cess)g(con)o(tains)f(the)h(data)f Fw(A)1085 2638 y Fi(0)1105 2631 y FC(,)h(but)g(after)e(the)i(broadcast) f(all)i(pro)q(cesses)75 2687 y(con)o(tain)15 b(it.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 73 81 73 80 bop 75 -100 a Fs(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FC(73)75 45 y Fp(4.4.1)49 b(Broadcast)90 178 y Fv(MPI)p 175 178 14 2 v 16 w(BCAST\()16 b(bu\013er,)f(count,)h (datat)o(yp)q(e,)g(ro)q(ot,)f(comm)e(\))117 255 y Fq(INOUT)62 b Fv(bu\013er)478 b Fq(starting)14 b(address)h(of)f(bu\013er)g(\(c)o (hoice\))117 330 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(bu\013er)i(\(in)o(teger\))117 404 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)i(\(handle\))117 479 y(IN)171 b Fv(ro)q(ot)492 b Fq(rank)14 b(of)f(broadcast)i(ro)q(ot)f(\(in)o(teger\))117 553 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 677 y Fo(int)47 b(MPI)p 269 677 15 2 v 17 w(Bcast\(void*)22 b(buf,)i(int)f(count,)g(MPI)p 1026 677 V 17 w(Datatype)g(datatype,)f (int)i(root,)393 734 y(MPI)p 468 734 V 17 w(Comm)f(comm)h(\))75 820 y(MPI)p 150 820 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,) g(COMM,)g(IERROR\))170 877 y()g(BUFFER\(*\))170 933 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)166 1020 y Fv(MPI)p 251 1020 14 2 v 16 w(BCAST)11 b FC(broadcasts)e(a)h (message)g(from)g(the)g(pro)q(cess)h(with)f(rank)21 b Fv(ro)q(ot)10 b FC(to)f(all)j(other)e(pro)q(cesses)75 1076 y(of)18 b(the)g(group.)28 b(It)18 b(is)h(called)h(b)o(y)e(all)h (mem)o(b)q(ers)f(of)g(group)g(using)g(the)h(same)e(argumen)o(ts)h(for) 35 b Fv(comm,)75 1133 y(ro)q(ot)16 b FC(and)g(matc)o(hing)g(argumen)o (ts)f(for)31 b Fv(count,)17 b(datat)o(yp)q(e)p FC(.)23 b(On)17 b(return)f(the)g(con)o(ten)o(ts)f(of)g(the)h(bu\013er)g(of)75 1189 y(the)f(pro)q(cess)h(with)f(rank)30 b Fv(ro)q(ot)15 b FC(is)h(con)o(tained)g(in)g(the)f(bu\013er)g(of)g(the)g(calling)i (pro)q(cess.)75 1310 y Fp(4.4.2)49 b(Gather)75 1443 y Fv(MPI)p 160 1443 V 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcount,)h (sendt)o(yp)q(e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e (comm\))117 1520 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1595 y(IN)171 b Fv(sendcount)383 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1669 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 1743 y(OUT)124 b Fv(recvbuf)434 b Fq(address)21 b(of)e(receiv)o(e)j (bu\013er)e(\(c)o(hoice,)i(signi\014can)o(t)d(only)g(at)905 1800 y(ro)q(ot\))117 1874 y(IN)171 b Fv(recvcount)389 b Fq(n)o(um)o(b)q(er)11 b(of)g(elemen)o(ts)g(for)g(an)o(y)g(single)g (receiv)o(e)i(\(in)o(teger,)f(sig-)905 1931 y(ni\014can)o(t)i(only)f (at)h(ro)q(ot\))117 2005 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fq(data)16 b(t)o(yp)q(e)g(of)f(recv)i(bu\013er)g(elemen)o(ts)f (\(signi\014can)o(t)f(only)g(at)905 2062 y(ro)q(ot\))f(\(handle\))117 2136 y(IN)171 b Fv(ro)q(ot)492 b Fq(rank)14 b(of)f(receiving)i(pro)q (cess)g(\(in)o(teger\))117 2210 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 2335 y Fo(int)23 b(MPI)p 245 2335 15 2 v 17 w(Gather\(void*)f(sendbuf,)h(int)h (sendcount,)e(MPI)p 1217 2335 V 17 w(Datatype)h(sendtype,)393 2391 y(void*)g(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 2391 V 17 w(Datatype)h(recvtype,)f(int)i(root,)393 2448 y(MPI)p 468 2448 V 17 w(Comm)f(comm\))75 2534 y(MPI)p 150 2534 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 2591 y(ROOT,)g(COMM,)h(IERROR\))170 2647 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2704 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 74 82 74 81 bop 75 -100 a FC(74)703 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y FC(Eac)o(h)h(pro)q(cess)g (\(including)i(the)e(ro)q(ot)f(pro)q(cess\))h(sends)g(the)g(con)o(ten)o (ts)g(of)f(its)h(send)h(bu\013er)f(to)f(the)75 102 y(ro)q(ot)i(pro)q (cess.)31 b(The)19 b(ro)q(ot)f(pro)q(cess)h(receiv)o(es)h(the)f (messages)f(at)g(stores)g(them)h(in)h(rank)e(order.)31 b(The)75 158 y(outcome)15 b(is)h(as)e(if)i(eac)o(h)f(of)g(the)g Fo(n)g FC(pro)q(cesses)h(in)g(the)f(group)g(had)h(executed)g(a)f(call)h (to)189 237 y Fo(MPI)p 264 237 15 2 v 16 w(Send)p FC(\()p Fo(sendbuf)p Fw(;)8 b Fo(sendcount)p Fw(;)f Fo(se)o(ndtype)p Fw(;)g Fo(roo)o(t)p Fw(;)h(:)o(::)o FC(\))p Fw(;)75 317 y FC(and)15 b(the)h(ro)q(ot)e(had)h(executed)i Fo(n)e FC(calls)h(to)189 396 y Fo(MPI)p 264 396 V 16 w(Recv)p FC(\()p Fo(recvbuf)p FC([)p Fo(i)p FC(])p Fw(;)8 b Fo(rec)o(vcount)p Fw(;)f Fo(rec)o(vtype)p Fw(;)g Fo(i)p Fw(;)g(::)o(:)p FC(\))p Fw(;)75 476 y FC(where,)189 555 y Fo(recvbuf)p FC([)p Fo(i)p FC(])j(=)j Fo(recvbuf)d FC(+)g Fo(i)g FA(\002)g Fo(recvcount)f FA(\002)i Fo(extent)p FC(\()p Fo(recvtype)p FC(\))p Fw(:)166 635 y FC(An)18 b(alternativ)o(e)h(description)h(is)e (that)g(the)g Fo(n)g FC(messages)g(sen)o(t)g(b)o(y)g(the)g(pro)q (cesses)h(in)g(the)f(group)75 691 y(are)c(concatenated)h(in)g(rank)f (order,)g(and)h(the)g(resulting)g(message)f(is)h(receiv)o(ed)h(b)o(y)e (the)h(ro)q(ot)f(as)g(if)h(b)o(y)f(a)75 748 y(call)i(to)f Fv(MPI)p 299 748 14 2 v 16 w(RECV\(recvbuf,)h(recvcount)g FA(\002)p Fv(n,)g(recvt)o(yp)q(e,)f(...\))p FC(.)166 804 y(The)g(receiv)o(e)h(bu\013er)g(is)f(ignored)h(for)f(all)h(non-ro)q (ot)f(pro)q(cesses.)166 860 y(General)21 b(deriv)o(ed)h(datat)o(yp)q (es)f(are)f(allo)o(w)o(ed)i(for)e(b)q(oth)42 b Fv(sendt)o(yp)q(e)24 b FC(and)42 b Fv(recvt)o(yp)q(e)p FC(.)c(The)21 b(t)o(yp)q(e)75 917 y(signature)15 b(of)30 b Fv(sendcount,)18 b(sendt)o(yp)q(e)g FC(on)d(pro)q(cess)g(i)h(m)o(ust)f(b)q(e)h(equal)g(to)e(the)h(t)o(yp)q (e)h(signature)f(of)30 b Fv(recv-)75 973 y(count,)14 b(recvt)o(yp)q(e)g FC(at)e(the)h(ro)q(ot.)19 b(Note)12 b(that)g(the)h(amoun)o(t)f(of)h(data)f(sen)o(t)h(m)o(ust)f(b)q(e)i (equal)f(to)g(the)g(amoun)o(t)75 1030 y(receiv)o(ed)20 b(\(pairwise)f(b)q(et)o(w)o(een)g(eac)o(h)f(pro)q(cess)h(and)g(the)g (ro)q(ot\).)28 b Fv(MPI)p 1298 1030 V 16 w(GA)l(THER)20 b FC(and)f(all)g(other)g(data)75 1086 y(mo)o(v)o(emen)o(t)10 b(collectiv)o(e)j(routines)f(mak)o(e)f(this)h(restriction)f(and)h(pro)o (vide)g(no)f(facilit)o(y)h(\(suc)o(h)g(as)e(the)i(status)75 1143 y(argumen)o(t)i(of)h Fv(MPI)p 415 1143 V 16 w(RECV)p FC(\))h(for)e(disco)o(v)o(ering)i(ho)o(w)f(m)o(uc)o(h)g(data)g(w)o(as)f (sen)o(t.)166 1199 y(All)h(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p FC(,)e(while)j(on)e (other)g(pro)q(cesses,)75 1256 y(only)19 b(argumen)o(ts)f Fv(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,)d(comm)e FC(are)i(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1312 y Fv(ro)q(ot)c FC(and)g Fv(comm)f FC(m)o(ust)g(ha)o(v)o(e)h(iden)o (tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)75 1416 y Fv(MPI)p 160 1416 V 16 w(GA)l(THERV\()24 b(sendbuf,)j(sendcount,)g (sendt)o(yp)q(e,)g(recvbuf,)e(recvcounts,)h(displs,)g(recvt)o(yp)q(e,)g (ro)q(ot,)75 1472 y(comm\))117 1549 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1622 y(IN)171 b Fv(sendcount)383 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1694 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 1766 y(OUT)124 b Fv(recvbuf)434 b Fq(address)21 b(of)e(receiv)o(e)j (bu\013er)e(\(c)o(hoice,)i(signi\014can)o(t)d(only)g(at)905 1823 y(ro)q(ot\))117 1895 y(IN)171 b Fv(recvcounts)372 b Fq(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(con)o (taining)e(the)i(n)o(um-)905 1951 y(b)q(er)20 b(of)e(elemen)o(ts)h (that)f(are)i(receiv)o(ed)g(from)d(eac)o(h)i(pro)q(cess)905 2008 y(\(signi\014can)o(t)14 b(only)f(at)h(ro)q(ot\))117 2080 y(IN)171 b Fv(displs)469 b Fq(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fk(i)f Fq(sp)q(eci\014es) 905 2137 y(the)h(displacemen)o(t)e(relativ)o(e)g(to)h Fl(recvbuf)i Fq(at)d(whic)o(h)h(to)g(place)905 2193 y(the)h(incoming)e (data)h(from)f(pro)q(cess)k Fk(i)d Fq(\(signi\014can)o(t)g(only)g(at) 905 2249 y(ro)q(ot\))117 2322 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fq(data)16 b(t)o(yp)q(e)g(of)f(recv)i(bu\013er)g(elemen)o(ts)f (\(signi\014can)o(t)f(only)g(at)905 2378 y(ro)q(ot\))f(\(handle\))117 2450 y(IN)171 b Fv(ro)q(ot)492 b Fq(rank)14 b(of)f(receiving)i(pro)q (cess)g(\(in)o(teger\))117 2523 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 2647 y Fo(int)23 b(MPI)p 245 2647 15 2 v 17 w(Gatherv\(void*)f(sendbuf,)h(int)h (sendcount,)e(MPI)p 1241 2647 V 17 w(Datatype)h(sendtype,)393 2704 y(void*)g(recvbuf,)g(int)h(*recvcounts,)e(int)h(*displs,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 75 83 75 82 bop 75 -100 a Fs(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FC(75)393 45 y Fo(MPI)p 468 45 15 2 v 17 w(Datatype)23 b(recvtype,)f(int)i(root,)f(MPI)p 1249 45 V 17 w(Comm)g(comm\))75 132 y(MPI)p 150 132 V 17 w(GATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f (RECVCOUNTS,)h(DISPLS,)393 188 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\)) 170 245 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 301 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h (RECVTYPE,)g(ROOT,)170 358 y(COMM,)h(IERROR)166 444 y Fv(MPI)p 251 444 14 2 v 16 w(GA)l(THERV)15 b FC(extends)g(the)f (functionalit)o(y)h(of)e Fv(MPI)p 1126 444 V 16 w(GA)l(THER)i FC(b)o(y)f(allo)o(wing)h(a)f(v)m(arying)g(coun)o(t)75 501 y(of)e(data)h(from)f(eac)o(h)g(pro)q(cess)i(\(since)f Fv(recvcounts)i FC(is)e(no)o(w)f(an)h(arra)o(y\),)e(and)i(also)g(allo)o (ws)g(more)f(\015exibili)q(t)o(y)75 557 y(as)j(to)f(where)i(the)f(data) g(is)g(placed)i(on)e(the)g(ro)q(ot)g(\(b)o(y)f(pro)o(viding)i(the)g (new)f(argumen)o(t,)f Fv(displs)p FC(\).)166 614 y(The)h(outcome)g(is)h (as)f(if)g(eac)o(h)h(pro)q(cess)f(has)g(sen)o(t)g(a)g(message)g(to)f (the)i(ro)q(ot,)189 709 y Fo(MPI)p 264 709 15 2 v 16 w(Send)p FC(\()p Fo(sendbuf)p Fw(;)8 b Fo(sendcount)p Fw(;)f Fo(se)o(ndtype)p Fw(;)g Fo(roo)o(t)p Fw(;)h(:)o(::)o FC(\))p Fw(;)75 804 y FC(and)15 b(the)h(ro)q(ot)e(executed)i Fo(n)f FC(receiv)o(es,)189 900 y Fo(MPI)p 264 900 V 16 w(Recv)p FC(\()p Fo(recvbuf)9 b FC(+)h Fo(disp)p FC([)p Fo(i)p FC(])f FA(\002)h Fo(extent)p FC(\()p Fo(recvtype)p FC(\))p Fw(;)e Fo(rec)o(vcounts)p FC([)o Fo(i)p FC(])p Fw(;)f Fo(rec)o(vtype)p Fw(;)g Fo(i)p Fw(;)g(::)o(:)p FC(\))p Fw(:)166 995 y FC(Messages)14 b(are)g(placed)i(in)f(the)g (receiv)o(e)g(bu\013er)g(of)f(the)g(ro)q(ot)g(pro)q(cess)h(in)g(rank)f (order,)g(that)g(is,)h(the)75 1051 y(data)d(sen)o(t)g(from)f(pro)q (cess)i Fo(j)f FC(is)h(placed)h(in)f(the)f Fo(j)p FC(-th)h(p)q(ortion)f (of)g(the)h(receiv)o(e)g(bu\013er)f Fv(recvbuf)h FC(on)g(pro)q(cess)75 1108 y Fv(ro)q(ot)p FC(.)29 b(The)18 b Fo(j)p FC(-th)g(p)q(ortion)h(of) f Fv(recvbuf)h FC(b)q(egins)g(at)f(o\013set)f Fv(displs[j])j FC(elemen)o(ts)f(\(in)g(terms)f(of)f Fv(recvt)o(yp)q(e)p FC(\))75 1164 y(in)o(to)e Fv(recvbuf)p FC(.)166 1221 y(The)i(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fv(sendcount,)j (sendt)o(yp)q(e)g FC(on)e(pro)q(cess)g Fo(i)f FC(m)o(ust)g(b)q(e)h (equal)g(to)f(the)75 1277 y(t)o(yp)q(e)h(signature)f(implied)j(b)o(y)e Fv(recvcounts[i],)h(recvt)o(yp)q(e)f FC(at)f(the)h(ro)q(ot)f(\(ho)o(w)o (ev)o(er,)f(the)i(t)o(yp)q(e)f(maps)h(ma)o(y)75 1334 y(b)q(e)f(di\013eren)o(t\).)166 1391 y(All)f(argumen)o(ts)f(to)f(the)h (function)h(are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p FC(,)e(while)j(on)e(other)g(pro)q(cesses,)75 1447 y(only)19 b(argumen)o(ts)f Fv(sendbuf,)k(sendcount,)g(sendt)o(yp)q (e,)g(ro)q(ot,)d(comm)e FC(are)i(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1504 y Fv(ro)q(ot)c FC(and)g Fv(comm)f FC(m)o(ust)g(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)f(on)f (all)i(pro)q(cesses.)166 1560 y(F)l(or)f(b)q(oth)h(functions,)g(the)g (sp)q(eci\014cation)h(of)f(coun)o(t\(s\),)e(t)o(yp)q(e\(s\),)h (\(displacemen)o(ts\),)i(should)f(not)75 1617 y(cause)e(an)o(y)g(lo)q (cation)h(on)f(the)h(ro)q(ot)e(to)h(b)q(e)g(written)h(more)e(than)i (once.)k(Suc)o(h)c(a)e(call)j(is)f(erroneous.)166 1674 y(W)l(e)f(illustrate)h(the)g(matc)o(hing)f(conditions)h(with)g(the)f (follo)o(wing)h(examples.)75 1797 y Fp(4.4.3)49 b(Examples)16 b(of)h(Usage)g(of)f(MPI)p 768 1797 V 18 w(GA)l(THER,)h(MPI)p 1091 1797 V 18 w(GA)l(THERV)75 1883 y Fv(Example)e(1)75 1970 y FC(Gather)f(100)h(in)o(ts)g(from)g(ev)o(ery)g(pro)q(cess)g(in)h (group)f(to)g(ro)q(ot.)k(See)d(\014gure)f(4.2.)170 2078 y Fo(MPI_Comm)23 b(comm;)170 2134 y(int)h(gsize,sendarray[100];)170 2191 y(int)g(root,)f(*rbuf;)170 2304 y(...)170 2417 y(/*)h(The)f (variable)g(comm)h(is)f(set)h(elsewhere)e(in)i(the)f(program)194 2473 y(*/)170 2530 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2586 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 2642 y(MPI_Gather\()f(sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(100,)f (MPI_INT,)g(root,)g(comm\);)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 76 84 76 83 bop 75 -100 a FC(76)708 b Fs(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 189 244 m 189 294 l gs 0.50 setgray fill gr gs col-1 s gr n 191.000 286.000 m 189.000 294.000 l 187.000 286.000 l gs 2 setlinejoin col-1 s gr n 289 244 m 259 294 l gs 0.50 setgray fill gr gs col-1 s gr n 264.831 288.169 m 259.000 294.000 l 261.401 286.111 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr $F2psEnd %%EndDocument endTexFig 186 511 a FC(Figure)g(4.2:)j(The)c(ro)q(ot)g(pro)q(cess)g(gathers)f (100)h Fo(int)p FC(s)f(from)h(eac)o(h)g(pro)q(cess)h(in)g(the)f(group.) 75 647 y Fv(Example)g(2)75 735 y FC(Do)d(the)h(same)g(as)f(in)i (previous)g(example,)f(but)g(use)h(a)e(deriv)o(ed)i(datat)o(yp)q(e.)19 b(Note)12 b(that)g(the)h(t)o(yp)q(e)g(cannot)75 792 y(b)q(e)19 b(the)f(en)o(tire)g(set)g(of)f Fo(gsize*100)23 b(int)p FC(s)18 b(since)h(t)o(yp)q(e)f(matc)o(hing)g(is)g(de\014ned)i(pairwise) f(b)q(et)o(w)o(een)f(the)75 848 y(ro)q(ot)c(and)i(eac)o(h)f(pro)q(cess) g(in)h(the)g(gather.)170 962 y Fo(MPI_Comm)23 b(comm;)170 1019 y(int)h(gsize,sendarray[100];)170 1075 y(int)g(root,)f(*rbuf;)170 1132 y(MPI_Datatype)g(rtype;)170 1245 y(...)170 1358 y(/*)h(The)f(variable)g(comm)h(is)f(set)h(elsewhere)e(in)i(the)f (program)194 1414 y(*/)170 1470 y(MPI_Comm_size\()f(comm,)i(&gsize\);) 170 1527 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o(\(int\)\);) 170 1583 y(MPI_Type_contiguous\()e(100,)h(MPI_INT,)g(&rtype)g(\);)170 1640 y(MPI_Type_commit\()f(rtype)h(\);)170 1696 y(MPI_Gather\()g (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(1,)f(rtype,)g(root,)g(comm\);) 75 1824 y Fv(Example)15 b(3)75 1912 y FC(No)o(w)e(ha)o(v)o(e)g(eac)o(h) h(pro)q(cess)g(send)g(100)f(in)o(ts)h(to)f(ro)q(ot,)g(but)g(place)i (eac)o(h)f(set)f(\(of)g(100\))f Fr(stride)i FC(in)o(ts)g(apart)f(at)75 1969 y(receiving)18 b(end.)24 b(Use)16 b Fv(MPI)p 547 1969 14 2 v 16 w(GA)l(THERV)i FC(and)e(the)h Fv(displs)g FC(argumen)o(t)f(to)f(ac)o(hiev)o(e)i(this)g(e\013ect.)23 b(Assume)75 2025 y Fw(str)q(ide)13 b FA(\025)g FC(100.)19 b(See)c(\014gure)h(4.3.)170 2139 y Fo(MPI_Comm)23 b(comm;)170 2195 y(int)h(gsize,sendarray[100];)170 2252 y(int)g(root,)f(*rbuf,)g (stride;)170 2308 y(int)h(*displs,i,*rcounts;)170 2421 y(...)170 2534 y(MPI_Comm_size\()e(comm,)i(&gsize\);)170 2591 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*stride*siz)o(eof\(int)o (\)\);)170 2647 y(displs)f(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o (nt\)\);)170 2704 y(rcounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 77 85 77 84 bop 75 -100 a Fs(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FC(77)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig3.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 189 244 m 204 294 l gs col-1 s gr n 203.617 285.763 m 204.000 294.000 l 199.786 286.912 l gs 2 setlinejoin col-1 s gr n 289 244 m 289 294 l gs col-1 s gr n 291.000 286.000 m 289.000 294.000 l 287.000 286.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 511 a FC(Figure)17 b(4.3:)22 b(The)16 b(ro)q(ot)g(pro)q(cess)h (gathers)f(100)g Fo(int)p FC(s)g(from)f(eac)o(h)i(pro)q(cess)g(in)g (the)g(group,)f(eac)o(h)h(set)f(is)75 568 y(placed)g Fo(stride)f FC(in)o(ts)g(apart.)170 701 y Fo(for)24 b(\(i=0;)f (i)f(SENDBUF\(*\),)g (RECVBUF\(*\))170 1469 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,) h(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR)166 1560 y Fv(MPI)p 251 1560 14 2 v 16 w(SCA)l(TTER)16 b FC(is)g(the)f(in)o(v)o(erse)h(op)q (eration)f(to)g Fv(MPI)p 1097 1560 V 15 w(GA)l(THER)p FC(.)166 1621 y(The)e(ro)q(ot)g(pro)q(cess)g(sends)h(a)f(message)g(to)g (eac)o(h)g(pro)q(cess)g(\(including)j(itself)t(\);)e(eac)o(h)f(pro)q (cess)h(stores)75 1677 y(the)j(incoming)h(message)e(in)i(its)f(receiv)o (e)h(bu\013er.)25 b(The)17 b(outcome)f(is)h(as)g(if)g(the)g(ro)q(ot)f (executed)i Fo(n)e FC(send)75 1734 y(op)q(erations,)189 1849 y Fo(MPI)p 264 1849 15 2 v 16 w(Send)p FC(\()p Fo(sendbuf)9 b FC(+)h Fo(i)g FA(\002)h Fo(sendcount)d FA(\002)j Fo(extent)p FC(\()p Fo(sendtype)p FC(\))p Fw(;)d Fo(s)o(endcoun)o(t)p Fw(;)f Fo(sendtype)o Fw(;)h Fo(i)o Fw(;)g(:)o(::)p FC(\))o Fw(;)75 1960 y FC(and)15 b(eac)o(h)h(pro)q(cess)f(executed)h(a)f (receiv)o(e,)189 2076 y Fo(MPI)p 264 2076 V 16 w(Recv)p FC(\()p Fo(recvbuf)p Fw(;)8 b Fo(recvcount)p Fw(;)f Fo(re)o(cvtype)p Fw(;)g Fo(i)p Fw(;)g(::)o(:)p FC(\))o Fw(:)166 2187 y FC(An)22 b(alternativ)o(e)f(description)i(is)f(that)f(the)g(ro)q(ot)g (send)h(a)f(message)g(with)h Fv(MPI)p 1594 2187 14 2 v 16 w(Send\(sendbuf,)75 2243 y(sendcount)p FA(\002)p Fv(n)q(,)j(sendt)o(yp)q(e,)h(...\))p FC(;)c(this)g(message)f(is)h (split)h(in)o(to)e Fo(n)g FC(equal)i(segmen)o(ts,)f(and)f(the)h Fw(i)14 b FA(\000)g Fw(th)75 2300 y FC(segmen)o(t)h(is)i(sen)o(t)e(to)g (the)h Fw(i)p FC(-th)g(pro)q(cess)g(in)g(the)g(group;)f(and)h(eac)o(h)g (pro)q(cess)g(receiv)o(es)h(this)f(message)f(as)75 2356 y(ab)q(o)o(v)o(e.)166 2417 y(The)j(t)o(yp)q(e)f(signature)h(asso)q (ciated)g(with)f Fv(sendcount,)k(sendt)o(yp)q(e)f FC(at)d(the)h(ro)q (ot)e(m)o(ust)h(b)q(e)h(equal)h(to)75 2473 y(the)f(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fv(recvcount,)h(recvt)o(yp)q(e)g FC(at)e(all)i(pro)q(cesses)f(\(ho)o(w)o(ev)o(er,)f(the)h(t)o(yp)q(e)75 2530 y(maps)d(ma)o(y)f(b)q(e)i(di\013eren)o(t\).)166 2591 y(All)f(argumen)o(ts)f(to)f(the)h(function)h(are)f(signi\014can)o (t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p FC(,)e(while)j(on)e(other)g(pro)q (cesses,)75 2647 y(only)f(argumen)o(ts)f Fv(recvbuf,)i(recvcount,)g (recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)f FC(are)h(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fv(ro)q(ot)75 2704 y FC(and)j Fv(comm)f FC(m)o(ust)g(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e (all)h(pro)q(cesses.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 83 91 83 90 bop 75 -100 a Fs(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FC(83)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SCA)l(TTERV\()21 b(sendbuf,)j(sendcounts)q(,)g(displs,)f(sendt)o (yp)q(e,)h(recvbuf,)f(recvcount,)f(recvt)o(yp)q(e,)h(ro)q(ot,)75 102 y(comm\))117 180 y Fq(IN)171 b Fv(sendbuf)428 b Fq(address)13 b(of)d(send)i(bu\013er)h(\(c)o(hoice,)f(signi\014can)o(t)e(only)h(at)g (ro)q(ot\))117 259 y(IN)171 b Fv(sendcounts)366 b Fq(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(sp)q(ecifying)g(the)g(n)o (um-)905 315 y(b)q(er)15 b(of)e(elemen)o(ts)h(to)g(send)h(to)e(eac)o(h) i(pro)q(cessor)117 393 y(IN)171 b Fv(displs)469 b Fq(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fk(i)f Fq(sp)q(eci\014es)905 450 y(the)20 b(displacemen)o(t)e (\(relativ)o(e)h(to)f Fl(sendbuf)j Fq(from)c(whic)o(h)i(to)905 506 y(tak)o(e)14 b(the)h(outgoing)d(data)i(to)g(pro)q(cess)h Fk(i)117 584 y Fq(IN)171 b Fv(sendt)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 663 y(OUT)124 b Fv(recvbuf)434 b Fq(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 741 y(IN)171 b Fv(recvcount)389 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 819 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fq(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 897 y(IN)171 b Fv(ro)q(ot)492 b Fq(rank)14 b(of)f(sending)h(pro)q(cess)i(\(in)o(teger\))117 976 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 1102 y Fo(int)23 b(MPI)p 245 1102 15 2 v 17 w(Scatterv\(void*)f (sendbuf,)h(int)g(*sendcounts,)g(int)g(*displs,)393 1158 y(MPI)p 468 1158 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e(int)i (recvcount,)393 1215 y(MPI)p 468 1215 V 17 w(Datatype)f(recvtype,)f (int)i(root,)f(MPI)p 1249 1215 V 17 w(Comm)g(comm\))75 1303 y(MPI)p 150 1303 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i (DISPLS,)g(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)393 1359 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 1416 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 1472 y(INTEGER)g(SENDCOUNTS\(*\),)f (DISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 1529 y(COMM,)h(IERROR)166 1617 y Fv(MPI)p 251 1617 14 2 v 16 w(SCA)l(TTERV)16 b FC(is)g(the)f(in)o(v)o(erse)h(op)q(eration)f (to)g Fv(MPI)p 1127 1617 V 16 w(GA)l(THERV)p FC(.)166 1675 y Fv(MPI)p 251 1675 V 16 w(SCA)l(TTERV)24 b FC(extends)f(the)g (functionalit)o(y)g(of)g Fv(MPI)p 1193 1675 V 15 w(SCA)l(TTER)h FC(b)o(y)f(allo)o(wing)g(a)f(v)m(arying)75 1731 y(coun)o(t)14 b(of)g(data)g(to)g(b)q(e)h(sen)o(t)f(to)f(eac)o(h)i(pro)q(cess)f (\(since)i Fv(sendcounts)h FC(is)e(no)o(w)f(an)g(arra)o(y\),)f(and)h (also)h(allo)o(ws)75 1787 y(more)21 b(\015exibili)q(t)o(y)j(as)d(to)g (where)h(the)g(data)f(is)i(tak)o(en)e(from)g(on)h(the)g(ro)q(ot)f(\(b)o (y)g(pro)o(viding)i(the)f(new)75 1844 y(argumen)o(t,)14 b Fv(displs)p FC(\).)166 1902 y(The)h(outcome)g(is)h(as)f(if)g(the)h (ro)q(ot)e(excuted)i Fo(n)f FC(send)h(op)q(erations,)189 2004 y Fo(MPI)p 264 2004 15 2 v 16 w(Send)p FC(\()p Fo(sendbuf)9 b FC(+)h Fo(displs)p FC([)p Fo(i)p FC(])f FA(\002)h Fo(extent)p FC(\()p Fo(sendtype)p FC(\))p Fw(;)e Fo(s)o(endcount)o(s)p FC([)p Fo(i)p FC(])o Fw(;)g Fo(s)o(endtype)o Fw(;)g Fo(i)p Fw(;)f(:)o(::)p FC(\))o Fw(;)75 2104 y FC(and)15 b(eac)o(h)h(pro)q (cess)f(executed)h(a)f(receiv)o(e,)189 2205 y Fo(MPI)p 264 2205 V 16 w(Recv)p FC(\()p Fo(recvbuf)p Fw(;)8 b Fo(recvcount)p Fw(;)f Fo(re)o(cvtype)p Fw(;)g Fo(i)p Fw(;)g(::)o(:)p FC(\))o Fw(:)166 2305 y FC(The)16 b(t)o(yp)q(e)g (signature)g(implied)i(b)o(y)e Fv(sendcount[i],)j(sendt)o(yp)q(e)g FC(at)c(the)h(ro)q(ot)f(m)o(ust)g(b)q(e)i(equal)g(to)e(the)75 2362 y(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fv(recvcount,)i (recvt)o(yp)q(e)f FC(at)f(pro)q(cess)g Fo(i)g FC(\(ho)o(w)o(ev)o(er,)f (the)i(t)o(yp)q(e)f(maps)h(ma)o(y)e(b)q(e)75 2418 y(di\013eren)o(t\).) 166 2476 y(All)i(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p FC(,)e(while)j(on)e (other)g(pro)q(cesses,)75 2533 y(only)f(argumen)o(ts)f Fv(recvbuf,)i(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)f FC(are)h(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fv(ro)q(ot)75 2589 y FC(and)j Fv(comm)f FC(m)o(ust)g(ha)o(v)o(e)h(iden) o(tical)i(v)m(alues)g(on)e(all)h(pro)q(cesses.)166 2647 y(F)l(or)g(b)q(oth)h(functions,)g(the)g(sp)q(eci\014cation)h(of)f(coun) o(t\(s\),)e(t)o(yp)q(e\(s\),)h(\(displacemen)o(ts\),)i(should)f(not)75 2704 y(cause)e(an)o(y)g(lo)q(cation)h(on)f(the)h(ro)q(ot)e(to)h(b)q(e)g (read)h(more)e(than)h(once.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 84 92 84 91 bop 75 -100 a FC(84)708 b Fs(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6630800 0 0 33943388 13682606 startTexFig 458 4 a %%BeginDocument: mycoll-fig7.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 373.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 414 m gs 1 -1 scale (sendbuf) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 89 244 l gs col-1 s gr n 91.075 251.981 m 89.000 244.000 l 94.587 250.065 l gs 2 setlinejoin col-1 s gr n 189 299 m 189 244 l gs col-1 s gr n 187.000 252.000 m 189.000 244.000 l 191.000 252.000 l gs 2 setlinejoin col-1 s gr n 249 299 m 289 244 l gs col-1 s gr n 282.677 249.294 m 289.000 244.000 l 285.912 251.646 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (other processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 138 522 a FC(Figure)g(4.7:)i(The)e(ro)q(ot)e(pro)q(cess)i(scatters)e (sets)h(of)g(100)f Fo(int)p FC(s)h(to)f(eac)o(h)i(pro)q(cess)f(in)h (the)f(group.)75 655 y Fp(4.4.5)49 b(Examples)16 b(of)h(Usage)g(of)f (MPI)p 768 655 15 2 v 18 w(SCA)l(TTER,)h(MPI)p 1115 655 V 19 w(SCA)l(TTERV)75 740 y Fv(Example)e(9)75 826 y FC(The)k(rev)o (erse)g(of)g(Example)h(1.)31 b(Scatter)19 b(sets)g(of)f(100)h(in)o(ts)g (from)f(the)h(ro)q(ot)g(to)f(eac)o(h)h(pro)q(cess)h(in)g(the)75 883 y(group.)g(See)15 b(\014gure)h(4.7.)170 988 y Fo(MPI_Comm)23 b(comm;)170 1045 y(int)h(gsize,*sendbuf;)170 1101 y(int)g(root,)f (rbuf[100];)170 1214 y(...)170 1327 y(/*)h(The)f(variable)g(comm)h(is)f (set)h(elsewhere)e(in)i(the)f(program)194 1383 y(*/)170 1440 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 1496 y(sendbuf)f(=)h (\(int)f(*\)malloc\(gsize*100*sizeof\(int)o(\)\);)170 1553 y(MPI_Scatter\()g(sendbuf,)f(100,)i(MPI_INT,)f(rbuf,)g(100,)g (MPI_INT,)g(root,)g(comm\);)75 1722 y Fv(Example)15 b(10)75 1808 y FC(The)21 b(rev)o(erse)f(of)g(Example)i(3.)35 b(The)21 b(ro)q(ot)f(pro)q(cess)h(scatters)e(sets)i(of)f(100)f(in)o(ts) i(to)f(the)h(other)f(pro-)75 1864 y(cesses,)j(but)f(the)f(sets)h(of)f (100)g(are)g Fr(stride)g FC(in)o(ts)h(apart)f(in)h(the)g(sending)h (bu\013er.)38 b(Requires)23 b(use)f(of)75 1921 y Fv(MPI)p 160 1921 14 2 v 16 w(SCA)l(TTERV)p FC(.)16 b(Assume)f Fw(str)q(ide)e FA(\025)g FC(100.)19 b(See)d(\014gure)f(4.8.)170 2026 y Fo(MPI_Comm)23 b(comm;)170 2083 y(int)h(gsize,*sendbuf;)170 2139 y(int)g(root,)f(rbuf[100],)g(i,)g(*displs,)g(*scounts;)170 2252 y(...)170 2365 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2421 y(sendbuf)f(=)h(\(int)f(*\)malloc\(gsize*stride*sizeof\()o (int\)\);)170 2478 y(displs)g(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o (nt\)\);)170 2534 y(scounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)170 2591 y(for)h(\(i=0;)f(i)g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2504 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 2591 y Fv(MPI)p 251 2591 14 2 v 16 w(ALLGA)l(THER)16 b FC(can)g(b)q(e)h(though)o(t)e(of)g(as)h Fv(MPI)p 1047 2591 V 15 w(GA)l(THER)p FC(,)h(but)f(where)g(all)h(pro)q(cesses)f (receiv)o(e)75 2647 y(the)d(result,)g(instead)g(of)f(just)g(the)h(ro)q (ot.)18 b(The)13 b Fo(j)p FC(-th)f(blo)q(c)o(k)h(of)f(data)g(sen)o(t)h (from)e(eac)o(h)i(pro)q(cess)g(is)g(receiv)o(ed)75 2704 y(b)o(y)i(ev)o(ery)g(pro)q(cess)h(and)f(placed)h(in)g(the)g Fo(j)p FC(-th)f(blo)q(c)o(k)h(of)f(the)g(bu\013er)g Fv(recvbuf)p FC(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 87 95 87 94 bop 75 -100 a Fs(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FC(87)166 45 y(The)15 b(t)o(yp)q(e)g(signature)g (asso)q(ciated)g(with)h Fv(sendcount,)h(sendt)o(yp)q(e)p FC(,)h(at)c(a)h(pro)q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 102 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fv(recvcount,)g(recvt)o(yp)q(e)g FC(at)f(an)o(y)f(other)h(pro)q(cess.) 166 158 y(Th)o(us,)i(the)g(outcome)g(of)g(a)f(call)j(to)d Fv(MPI)p 881 158 14 2 v 16 w(ALLGA)l(THER\(...\))25 b FC(is)17 b(as)g(if)h(all)g(pro)q(cesses)f(executed)h Fo(n)75 214 y FC(calls)e(to)189 305 y Fo(MPI)p 264 305 15 2 v 16 w(GATHER)p FC(\()p Fo(sendbuf)p Fw(;)8 b Fo(sendcoun)o(t)p Fw(;)f Fo(sendtype)o Fw(;)h Fo(r)o(ecvbuf)p Fw(;)f Fo(rec)o(vcount)p Fw(;)g Fo(rec)o(vtype)p Fw(;)g Fo(root)o Fw(;)h Fo(c)o(omm)p FC(\))p Fw(;)75 395 y FC(for)15 b Fo(root)d FC(=)h Fo(0)p Fw(;)8 b FA(\001)g(\001)g(\001)t Fw(;)g Fo(n)h FA(\000)i Fo(1)p FC(.)75 499 y Fv(MPI)p 160 499 14 2 v 16 w(ALLGA)l(THERV\()g (sendbuf,)i(sendcount,)h(sendt)o(yp)q(e,)f(recvbuf,)f(recvcounts,)g (displs,)g(recvt)o(yp)q(e,)g(comm\))117 632 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 707 y(IN)171 b Fv(sendcount)383 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in) o(teger\))117 781 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 856 y(OUT)124 b Fv(recvbuf)434 b Fq(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 930 y(IN)171 b Fv(recvcounts)372 b Fq(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(con)o(taining)e(the)i(n)o(um-)905 987 y(b)q(er)15 b(of)e(elemen)o(ts)h(that)g(are)g(receiv)o(ed)h(from)e (eac)o(h)h(pro)q(cess)117 1061 y(IN)171 b Fv(displs)469 b Fq(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fk(i)f Fq(sp)q(eci\014es)905 1118 y(the)f(displacemen)o (t)e(\(relativ)o(e)g(to)h Fl(recvbuf)i Fq(at)d(whic)o(h)h(to)g(place) 905 1174 y(the)j(incoming)c(data)j(from)e(pro)q(cess)k Fk(i)117 1248 y Fq(IN)171 b Fv(recvt)o(yp)q(e)414 b Fq(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\)) 117 1323 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 1447 y Fo(int)23 b(MPI)p 245 1447 15 2 v 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1447 V 16 w(Datatype)g(sendtype,)393 1504 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1560 y(MPI)p 468 1560 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1560 V 17 w(Comm)h(comm\))75 1647 y(MPI)p 150 1647 V 17 w (ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNTS,)f(DISPLS,)393 1703 y(RECVTYPE,)h(COMM,)g(IERROR\))170 1760 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1816 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h (RECVTYPE,)g(COMM,)170 1872 y(IERROR)166 1959 y Fv(MPI)p 251 1959 14 2 v 16 w(ALLGA)l(THERV)11 b FC(can)g(b)q(e)g(though)o(t)e (of)h(as)g Fv(MPI)p 1044 1959 V 16 w(GA)l(THERV)p FC(,)h(but)g(where)f (all)h(pro)q(cesses)g(receiv)o(e)75 2015 y(the)i(result,)g(instead)g (of)f(just)g(the)h(ro)q(ot.)18 b(The)13 b Fo(j)p FC(-th)f(blo)q(c)o(k)h (of)f(data)g(sen)o(t)h(from)e(eac)o(h)i(pro)q(cess)g(is)g(receiv)o(ed) 75 2072 y(b)o(y)j(ev)o(ery)f(pro)q(cess)h(and)g(placed)h(in)f(the)g Fo(j)p FC(-th)g(blo)q(c)o(k)g(of)f(the)h(bu\013er)g Fv(recvbuf)p FC(.)21 b(These)16 b(blo)q(c)o(ks)h(need)f(not)75 2128 y(all)g(b)q(e)g(the)f(same)g(size.)166 2185 y(The)g(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fv(sendcount,)j(sendt)o(yp)q(e)p FC(,)f(at)d(pro)q(cess)h Fo(j)g FC(m)o(ust)f(b)q(e)i(equal)g(to)75 2241 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fv(recvcounts[j],)h(recvt)o(yp)q(e)f FC(at)e(an)o(y)h(other)g(pro)q (cess.)166 2298 y(The)g(outcome)g(is)h(as)f(if)g(all)h(pro)q(cesses)g (executed)g(calls)g(to)189 2388 y Fo(MPI)p 264 2388 15 2 v 16 w(GATHERV)p FC(\()p Fo(sendbuf)p Fw(;)8 b Fo(sendcou)o(nt)p Fw(;)f Fo(sendtyp)o(e)p Fw(;)g Fo(recvbuf)o Fw(;)h Fo(di)o(spls)p Fw(;)f Fo(recvc)o(ounts)p Fw(;)g Fo(recv)o(type)p Fw(;)189 2489 y Fo(root)p Fw(;)h Fo(comm)p FC(\))p Fw(;)75 2591 y FC(for)15 b Fo(root)d FC(=)h Fo(0)p Fw(;)8 b FA(\001)g(\001)g(\001)t Fw(;)g Fo(n)h FA(\000)i Fo(1)p FC(.)166 2647 y(F)l(or)i(b)q(oth)g Fv(MPI)p 436 2647 14 2 v 16 w(ALLGA)l(THER)i FC(and)e Fv(MPI)p 890 2647 V 16 w(ALLGA)l(THERV)p FC(,)h(all)h(argumen)o(ts)e (on)g(all)h(pro)q(cesses)g(are)75 2704 y(signi\014can)o(t.)21 b(The)15 b(argumen)o(t)g Fv(comm)e FC(m)o(ust)i(ha)o(v)o(e)f(iden)o (tical)j(v)m(alues)g(on)e(all)h(pro)q(cesses.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 88 96 88 95 bop 75 -100 a FC(88)703 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fp(4.4.7)49 b(All-to-All)19 b(Scatter/Gather)75 221 y Fv(MPI)p 160 221 14 2 v 16 w(ALL)l(TO)o(ALL\(sendbuf,)d(sendcount,)i(sendt)o(yp)q (e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(comm\))117 320 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 438 y(IN)171 b Fv(sendcount)383 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 557 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 676 y(OUT)124 b Fv(recvbuf)434 b Fq(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 795 y(IN)171 b Fv(recvcount)389 b Fq(n)o(um)o(b)q(er)17 b(of)f(elemen)o(ts) h(receiv)o(ed)i(from)c(an)o(y)i(pro)q(cess)i(\(in)o(te-)905 851 y(ger\))117 970 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fq(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 1089 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 1235 y Fo(int)23 b(MPI)p 245 1235 15 2 v 17 w(Alltoall\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1235 V 17 w(Datatype)f(sendtype,)393 1292 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1292 V 17 w(Datatype)h(recvtype,)393 1348 y(MPI)p 468 1348 V 17 w(Comm)g(comm\))75 1456 y(MPI)p 150 1456 V 17 w(ALLTOALL\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,) h(RECVCOUNT,)g(RECVTYPE,)393 1513 y(COMM,)g(IERROR\))170 1569 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1626 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 1734 y Fv(MPI)p 251 1734 14 2 v 16 w(ALL)l(TO)o(ALL)18 b FC(is)h(an)g(extension)h(of)e Fv(MPI)p 962 1734 V 16 w(ALLGA)l(THER)i FC(to)e(the)h(case)g(where)g(eac)o(h)g(pro)q(cess)75 1791 y(sends)d(distinct)h(data)d(to)h(eac)o(h)h(of)f(the)g(receiv)o (ers.)22 b(The)15 b Fo(j)p FC(-th)h(blo)q(c)o(k)g(sen)o(t)f(from)g(pro) q(cess)h Fo(i)f FC(is)h(receiv)o(ed)75 1847 y(b)o(y)f(pro)q(cess)h Fo(j)f FC(and)g(is)h(placed)g(in)g(the)f Fo(i)p FC(-th)h(blo)q(c)o(k)f (of)g Fv(recvbuf)p FC(.)166 1925 y(The)g(t)o(yp)q(e)g(signature)g(asso) q(ciated)g(with)h Fv(sendcount,)h(sendt)o(yp)q(e)p FC(,)h(at)c(a)h(pro) q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 1982 y(the)f(t)o(yp)q(e)h (signature)f(asso)q(ciated)g(with)h Fv(recvcount,)g(recvt)o(yp)q(e)g FC(at)f(an)o(y)f(other)h(pro)q(cess.)166 2060 y(The)e(outcome)g(is)h (as)e(if)i(eac)o(h)f(pro)q(cess)g(executed)h(a)f(send)h(to)e(eac)o(h)h (pro)q(cess)h(\(itself)f(included\))j(with)75 2116 y(a)f(call)h(to)189 2319 y Fo(MPI)p 264 2319 15 2 v 16 w(Send)p FC(\()p Fo(sendbuf)9 b FC(+)h Fo(i)g FA(\002)h Fo(sendcount)d FA(\002)j Fo(extent)p FC(\()p Fo(sendtype)p FC(\))p Fw(;)d Fo(s)o(endcoun)o(t)p Fw(;)f Fo(sendtype)o Fw(;)h Fo(i)o Fw(;)g(:)o(::)p FC(\))o Fw(;)75 2501 y FC(and)15 b(a)g(receiv)o(e)h(from)f(ev)o(ery)g(other)g (pro)q(cess)g(with)h(a)f(call)h(to)189 2704 y Fo(MPI)p 264 2704 V 16 w(Recv)p FC(\()p Fo(recvbuf)9 b FC(+)h Fo(i)g FA(\002)h Fo(recvcount)d FA(\002)j Fo(extent)p FC(\()p Fo(recvtype)p FC(\))p Fw(;)d Fo(r)o(ecvcoun)o(t)p Fw(;)f Fo(i)p Fw(;)h(:)o(::)o FC(\))p Fw(:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 89 97 89 96 bop 75 -100 a Fs(4.4.)34 b(D)o(A)l(T)l(A)15 b(MO)o(VE)g (FUNCTIONS)1077 b FC(89)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V\(sendbuf,)13 b(sendcounts)q(,)h(sdispls,)g (sendt)o(yp)q(e,)g(recvbuf,)e(recvcounts,)i(rdispls,)e(recvt)o(yp)q(e,) 75 102 y(comm\))117 180 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 259 y(IN)171 b Fv(sendcounts)366 b Fq(in)o(teger)20 b(arra)o(y)f(equal) h(to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 315 y(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(to)h(send)h(to)f(eac)o(h)g (pro)q(cessor)117 393 y(IN)171 b Fv(sdispls)452 b Fq(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fk(j)f Fq(sp)q(eci\014es)905 450 y(the)20 b(displacemen)o(t)e (\(relativ)o(e)h(to)f Fl(sendbuf)j Fq(from)c(whic)o(h)i(to)905 506 y(tak)o(e)14 b(the)h(outgoing)d(data)i(destined)h(for)e(pro)q(cess) j Fk(j)117 585 y Fq(IN)171 b Fv(sendt)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 663 y(OUT)124 b Fv(recvbuf)434 b Fq(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 741 y(IN)171 b Fv(recvcounts)372 b Fq(in)o(teger)20 b(arra)o(y)f(equal)h (to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 798 y(maxim)n(um)d(n)o(um)o(b)q(er)j(of)f(elemen)o(ts)i(that)g(can)f(b)q(e) i(receiv)o(ed)905 854 y(from)12 b(eac)o(h)j(pro)q(cessor)117 933 y(IN)171 b Fv(rdispls)453 b Fq(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fk(i)f Fq(sp)q(eci\014es) 905 989 y(the)f(displacemen)o(t)e(\(relativ)o(e)g(to)h Fl(recvbuf)i Fq(at)d(whic)o(h)h(to)g(place)905 1046 y(the)j(incoming)c (data)j(from)e(pro)q(cess)k Fk(i)117 1124 y Fq(IN)171 b Fv(recvt)o(yp)q(e)414 b Fq(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j (bu\013er)f(elemen)o(ts)e(\(handle\))117 1202 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 1328 y Fo(int)23 b(MPI)p 245 1328 15 2 v 17 w(Alltoallv\(void*)f (sendbuf,)h(int)g(*sendcounts,)f(int)i(*sdispls,)393 1385 y(MPI)p 468 1385 V 17 w(Datatype)f(sendtype,)f(void*)i(recvbuf,)e (int)i(*recvcounts,)393 1441 y(int)g(*rdispls,)e(MPI)p 802 1441 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1441 V 17 w(Comm)i(comm\))75 1529 y(MPI)p 150 1529 V 17 w (ALLTOALLV\(SENDBUF,)d(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h(RECVBUF,)g (RECVCOUNTS,)393 1586 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g(IERROR\))170 1642 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1699 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 1755 y(RECVTYPE,)h(COMM,)g (IERROR)166 1843 y Fv(MPI)p 251 1843 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V)12 b FC(adds)g(\015exibili)q(t)o(y)j(to)c Fv(MPI)p 959 1843 V 16 w(ALL)l(TO)o(ALL)h FC(in)h(that)f(the)h(lo)q(cation)g(of) f(data)g(for)g(the)75 1900 y(send)j(is)f(sp)q(eci\014ed)j(b)o(y)d Fv(sdispls)i FC(and)e(the)g(lo)q(cation)h(of)f(the)g(placemen)o(t)h(of) f(the)g(data)g(on)g(the)g(receiv)o(e)h(side)75 1956 y(is)h(sp)q (eci\014ed)h(b)o(y)e Fv(rdispls)p FC(.)166 2014 y(The)i Fo(j)p FC(-th)g(blo)q(c)o(k)h(sen)o(t)e(from)g(pro)q(cess)i Fo(i)e FC(is)i(receiv)o(ed)g(b)o(y)f(pro)q(cess)g Fo(j)g FC(and)g(is)g(placed)h(in)g(the)f Fo(i)p FC(-th)75 2071 y(blo)q(c)o(k)f(of)f Fv(recvbuf)p FC(.)20 b(These)c(blo)q(c)o(ks)g (need)g(not)f(all)h(ha)o(v)o(e)f(the)g(same)g(size.)166 2129 y(The)i(t)o(yp)q(e)h(signature)f(asso)q(ciated)h(with)f Fv(sendcount[)q(j],)j(sendt)o(yp)q(e)g FC(at)d(pro)q(cess)g Fo(i)g FC(m)o(ust)g(b)q(e)h(equal)75 2185 y(to)d(the)g(t)o(yp)q(e)g (signature)g(asso)q(ciated)h(with)f Fv(recvcount[i],)i(recvt)o(yp)q(e)f FC(at)e(pro)q(cess)i Fo(j)p FC(.)166 2243 y(The)f(outcome)g(is)h(as)f (if)g(eac)o(h)h(pro)q(cess)f(sen)o(t)g(a)g(message)g(to)f(eac)o(h)h (other)g(pro)q(cess)h(with,)189 2345 y Fo(MPI)p 264 2345 15 2 v 16 w(Send)p FC(\()p Fo(sendbuf)9 b FC(+)h Fo(displs)p FC([)p Fo(i)p FC(])p Fw(;)e Fo(sendcount)o(s)p FC([)p Fo(i)p FC(])o Fw(;)g Fo(s)o(endtype)o Fw(;)g Fo(i)p Fw(;)f(:)o(::)p FC(\))o Fw(;)75 2445 y FC(and)15 b(receiv)o(ed)i(a)e(message)f(from)h (ev)o(ery)g(other)g(pro)q(cess)g(with)h(a)f(call)h(to)189 2547 y Fo(MPI)p 264 2547 V 16 w(Recv)p FC(\()p Fo(recvbuf)9 b FC(+)h Fo(displs)p FC([)p Fo(i)p FC(])p Fw(;)e Fo(recvcount)o(s)p FC([)p Fo(i)p FC(])o Fw(;)g Fo(r)o(ecvtype)o Fw(;)g Fo(i)p Fw(;)f(:)o(::)p FC(\))o Fw(:)166 2647 y FC(F)l(or)21 b(b)q(oth)g Fv(MPI)p 452 2647 14 2 v 16 w(ALL)l(TO)o(ALL)g FC(and)h Fv(MPI)p 885 2647 V 16 w(ALL)l(TO)o(ALL)-5 b(V)p FC(,)20 b(all)j(argumen)o(ts)d(on)i(all)g(pro)q(cesses)g(are)75 2704 y(signi\014can)o(t.)f(The)15 b(argumen)o(t)g Fv(comm)e FC(m)o(ust)i(ha)o(v)o(e)f(iden)o(tical)j(v)m(alues)g(on)e(all)h(pro)q (cesses.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 90 98 90 97 bop 75 -100 a FC(90)703 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Fn(Implemen)o(tati)o(on)h (note:)54 b Fq(Although)18 b(the)h(discussion)g(of)e(collectiv)o(e)i (comm)o(unicati)o(on)d(in)h(terms)i(of)75 102 y(p)q(oin)o(t)e(to)g(p)q (oin)o(t)f(op)q(eration)h(implies)f(that)h(eac)o(h)h(message)e(is)i (transfered)g(directly)g(from)d(sender)k(to)e(receiv)o(er,)75 158 y(implemen)o(tatio)o(ns)11 b(ma)o(y)g(use)j(a)f(tree)h(comm)o (unication)c(pattern,)k(where)g(messages)f(are)h(forw)o(arded)f(b)o(y)g (in)o(terme-)75 214 y(diate)i(no)q(des)h(where)g(they)f(are)h(split)e (\(for)h(scatter\))i(or)d(concatenated)j(\(for)e(gather\),)g(if)f(this) h(is)g(more)f(e\016cien)o(t.)75 495 y Ft(4.5)59 b(Global)20 b(Compute)e(Op)r(erations)75 597 y FC(The)13 b(functions)g(in)h(this)f (section)h(p)q(erform)e(one)h(of)f(the)h(follo)o(wing)g(op)q(erations)g (across)f(all)i(the)f(mem)o(b)q(ers)75 653 y(of)i(a)g(group:)189 738 y(global)g(max)g(on)g(in)o(teger)h(and)f(\015oating)g(p)q(oin)o(t)h (data)e(t)o(yp)q(es)189 828 y(global)h(min)h(on)g(in)o(teger)f(and)g (\015oating)g(p)q(oin)o(t)h(data)f(t)o(yp)q(es)189 918 y(global)g(sum)h(on)f(in)o(teger)g(and)h(\015oating)f(p)q(oin)o(t)g (data)g(t)o(yp)q(es)189 1008 y(global)g(pro)q(duct)h(on)f(in)o(teger)h (and)f(\015oating)g(p)q(oin)o(t)h(data)e(t)o(yp)q(es)189 1098 y(global)h(AND)h(on)f(logical)h(and)f(in)o(teger)h(data)e(t)o(yp)q (es)189 1189 y(global)h(OR)h(on)g(logical)g(and)f(in)o(teger)h(data)e (t)o(yp)q(es)189 1279 y(global)h(X)o(OR)h(on)f(logical)i(and)e(in)o (teger)h(data)e(t)o(yp)q(es)189 1369 y(rank)h(of)f(pro)q(cess)i(with)f (maxim)o(um)g(v)m(alue)189 1459 y(rank)g(of)f(pro)q(cess)i(with)f (minim)o(um)i(v)m(alue)189 1549 y(user)e(de\014ned)i(\(asso)q(ciativ)o (e\))d(op)q(eration)189 1640 y(user)h(de\014ned)i(\(asso)q(ciativ)o(e)e (and)g(comm)o(utativ)o(e\))f(op)q(eration)75 1760 y Fp(4.5.1)49 b(Reduce)75 1893 y Fv(MPI)p 160 1893 14 2 v 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h(op,)e(ro)q(ot,)f (comm\))117 1970 y Fq(IN)171 b Fv(sendbuf)428 b Fq(address)15 b(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 2043 y(OUT)124 b Fv(recvbuf)434 b Fq(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o (hoice,)i(signi\014can)o(t)d(only)g(at)905 2100 y(ro)q(ot\))117 2173 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 2246 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)12 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)h(bu\013er)h(\(basic)f(t)o(yp)q(es)h(only\)) 905 2303 y(\(handle\))117 2376 y(IN)171 b Fv(op)525 b Fq(op)q(eration)14 b(\(state\))117 2449 y(IN)171 b Fv(ro)q(ot)492 b Fq(rank)14 b(of)f(ro)q(ot)h(pro)q(cess)i(\(in)o(teger\))117 2523 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 2647 y Fo(int)23 b(MPI)p 245 2647 15 2 v 17 w(Reduce\(void*)f(sendbuf,) h(void*)g(recvbuf,)g(int)h(count,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2704 V 17 w(Op)i(op,)f(int)h(root,)f(MPI)p 1505 2704 V 17 w(Comm)g(comm\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 91 99 91 98 bop 75 -100 a Fs(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FC(91)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(REDUCE\(SENDBUF,)22 b(RECVBUF,)g(COUNT,)h(DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR\))170 102 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 158 y(INTEGER)g(COUNT,) g(DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR)166 246 y FC(Com)o(bines)c(the)f(v)m(alues)i(pro)o(vided)f(in)g(the)g(send)g (bu\013er)f(of)g(eac)o(h)g(pro)q(cess)h(in)g(the)g(group,)f(using)75 302 y(the)d(op)q(eration)30 b Fv(op)p FC(,)15 b(and)g(returns)g(the)g (com)o(bined)h(v)m(alue)h(in)f(the)f(receiv)o(e)h(bu\013er)f(of)f(the)h (pro)q(cess)h(with)75 358 y(rank)32 b Fv(ro)q(ot)p FC(.)21 b(Eac)o(h)16 b(pro)q(cess)g(can)g(pro)o(vide)h(one)f(v)m(alue,)h(or)e (a)h(sequence)h(of)e(v)m(alues,)i(in)g(whic)o(h)g(case)f(the)75 415 y(com)o(bine)j(op)q(eration)f(is)h(executed)g(elemen)o(t-wise)g(on) g(eac)o(h)f(en)o(try)f(of)h(the)g(sequence.)30 b(F)l(or)17 b(example,)75 471 y(if)f(the)f(op)q(eration)h(is)g Fl(MPI)p 522 471 13 2 v 14 w(MAX)f FC(and)h(the)f(send)h(bu\013er)g(con)o(tains) f(t)o(w)o(o)f(\015oating)h(p)q(oin)o(t)h(n)o(um)o(b)q(ers,)g(then)75 528 y(recvbuf\(1\))h(=)h(global)h(max\(sendbuf\(1\)\))d(and)i (recvbuf\(2\))g(=)g(global)g(max\(sendbuf\(2\)\).)26 b(All)19 b(send)75 584 y(bu\013ers)d(should)i(de\014ne)g(sequences)f (of)f(equal)i(length)f(of)f(en)o(tries)h(all)g(of)f(the)h(same)f(data)g (t)o(yp)q(e,)h(where)75 641 y(the)d(t)o(yp)q(e)h(is)f(a)g Fx(basic)h FC(MPI)f(datat)o(yp)q(e)g(and)h(one)f(of)g(those)g(allo)o(w) o(ed)g(for)g(op)q(erands)h(of)27 b Fv(op)15 b FC(\(see)f(b)q(elo)o (w\).)75 697 y(F)l(or)f(all)j(op)q(erations)e(the)g(n)o(um)o(b)q(er)g (of)g(elemen)o(ts)h(in)g(the)f(send)h(bu\013er)f(are)g(the)g(same)g(as) f(for)h(the)g(receiv)o(e)75 754 y(bu\013ers.)22 b(F)l(or)16 b(all)h(op)q(erations,)e(the)i(t)o(yp)q(e)f(of)f(elemen)o(ts)i(in)g (the)f(send)g(bu\013er)g(are)g(the)g(same)g(as)f(for)h(the)75 810 y(receiv)o(e)g(bu\013ers.)166 868 y(The)f(op)q(eration)g(de\014ned) h(b)o(y)30 b Fv(op)15 b FC(is)g(asso)q(ciativ)o(e)g(and)g(comm)o (utativ)o(e,)f(and)h(the)g(implemen)o(tation)75 924 y(can)j(tak)o(e)f (adv)m(an)o(tage)g(of)g(asso)q(ciativit)o(y)h(and)g(comm)o(utativit)o (y)f(in)h(order)f(to)g(c)o(hange)h(order)f(of)h(ev)m(alu-)75 981 y(ation.)30 b(The)19 b(routine)g(is)g(called)h(b)o(y)f(all)g(group) g(mem)o(b)q(ers)f(using)i(the)e(same)h(argumen)o(ts)f(for)36 b Fv(count,)75 1037 y(datat)o(yp)q(e,)16 b(op,)f(ro)q(ot)g FC(and)31 b Fv(comm)p FC(.)166 1095 y(W)l(e)15 b(list)h(b)q(elo)o(w)g (the)f(supp)q(orted)h(options)f(for)30 b Fv(op)p FC(.)117 1198 y Fl(MPI)p 194 1198 V 14 w(MAX)649 b FC(maxim)o(um)117 1255 y Fl(MPI)p 194 1255 V 14 w(MIN)664 b FC(minim)o(um)117 1311 y Fl(MPI)p 194 1311 V 14 w(SUM)653 b FC(sum)117 1368 y Fl(MPI)p 194 1368 V 14 w(PROD)626 b FC(pro)q(duct)117 1424 y Fl(MPI)p 194 1424 V 14 w(LAND)632 b FC(logical)16 b(and)117 1481 y Fl(MPI)p 194 1481 V 14 w(BAND)626 b FC(bit-wise)16 b(and)117 1538 y Fl(MPI)p 194 1538 V 14 w(LOR)661 b FC(logical)16 b(or)117 1594 y Fl(MPI)p 194 1594 V 14 w(BOR)655 b FC(bit-wise)16 b(or)117 1651 y Fl(MPI)p 194 1651 V 14 w(LX)o(OR)634 b FC(logical)16 b(xor)117 1707 y Fl(MPI)p 194 1707 V 14 w(BX)o(OR)628 b FC(bit-wise)16 b(xor)117 1764 y Fl(MPI)p 194 1764 V 14 w(MAXLOC)569 b FC(maxim)o(um)15 b(v)m(alue)h(and)g(rank)f(of)f(pro)q (cess)i(with)f(it)117 1820 y Fl(MPI)p 194 1820 V 14 w(MINLOC)584 b FC(minim)o(um)16 b(v)m(alue)g(and)g(rank)f(of)g(pro)q(cess)g(with)h (it)166 1904 y(The)25 b Fl(MPI)p 346 1904 V 14 w(MINLOC)13 b FC(\()e Fl(MPI)p 634 1904 V 14 w(MAXLOC)p FC(\))h(op)q(erations)h (return)g(b)q(oth)g(minim)o(um)h(\(maxim)o(um\))e(v)m(alues)75 1961 y(and)17 b(the)h(ranks)f(of)g(pro)q(cesses)g(con)o(taining)h (those)f(v)m(alues.)28 b(The)17 b(p)q(oten)o(tially)i(mixed-t)o(yp)q(e) f(nature)f(of)75 2017 y(the)g(output)g(bu\013er)g(is)g(a)g(concern,)g (so)g(MPI)g(treats)f(the)h(bu\013ers)g(uniformly)h(and)f(co)q(erces)g (the)g(ranks)75 2074 y(to)j(the)h(same)f(t)o(yp)q(e)h(as)f(the)h(v)m (alues.)37 b(When)j Fl(MPI)p 995 2074 V 14 w(MINLOC)21 b FC(or)39 b Fl(MPI)p 1342 2074 V 14 w(MAXLOC)20 b FC(are)g(in)o(v)o (ok)o(ed,)i(the)75 2130 y(input)e(bu\013er)f(should)i(con)o(tain)e Fw(m)g FC(elemen)o(ts)h(of)f(a)g(data)f(t)o(yp)q(e)i(to)e(whic)o(h)i (the)f(op)q(eration)37 b Fl(MPI)p 1785 2130 V 15 w(MIN)75 2187 y FC(or)c Fl(MPI)p 226 2187 V 14 w(MAX)17 b FC(can)h(b)q(e)g (applied,)h(follo)o(w)o(ed)f(b)o(y)g(space)f(for)g(another)g Fw(m)g FC(elemen)o(ts)i(of)e(the)g(same)g(t)o(yp)q(e.)75 2243 y(In)o(ternally)l(,)h(the)e(function)h(will)h(co)q(erce)f(the)g (rank)f(of)g(the)g(calling)i(pro)q(cess)f(\(an)f(in)o(teger)g(t)o(yp)q (e\))g(to)g(the)75 2299 y(t)o(yp)q(e)g(of)g(the)g(data)f(v)m(alues,)i (and)f(carry)g(these)g(v)m(alues)h(along)f(during)h(the)f(reduction.)23 b(The)17 b(op)q(eration)75 2356 y(returns)f(at)f(the)i(ro)q(ot)e(the)h Fw(m)g FC(minim)o(um)h(\(or)e(maxim)o(um\))h(v)m(alues,)h(follo)o(w)o (ed)f(b)o(y)g(the)h Fw(m)e FC(ranks)h(of)g(the)75 2412 y(pro)q(cesses)k(con)o(taining)g(these)f(v)m(alues.)34 b(T)l(o)19 b(reco)o(v)o(er)f(the)i(ranks)f(of)g(the)g(pro)q(cesses)h (as)f(in)o(tegers,)h(the)75 2469 y(second)c(set)f(of)f(v)m(alues)j(can) e(no)o(w)g(b)q(e)h(copied)g(out)f(of)g(the)g(bu\013er)g(and)h(co)q (erced)g(to)e(in)o(teger)h(v)m(alues.)166 2526 y(The)g(op)q(eration)h (that)e(de\014nes)30 b Fl(MPI)p 801 2526 V 15 w(MAXLOC)14 b FC(is)189 2597 y Fg( )242 2641 y Fw(u)247 2697 y(i)289 2597 y Fg(!)332 2669 y FA(\016)365 2597 y Fg( )418 2641 y Fw(v)420 2697 y(j)463 2597 y Fg(!)508 2669 y FC(=)556 2597 y Fg( )610 2641 y Fw(w)614 2697 y(k)664 2597 y Fg(!)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 92 100 92 99 bop 75 -100 a FC(92)703 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y FC(where)189 131 y Fw(w)d FC(=)g(max\()p Fw(u;)8 b(v)r FC(\))75 217 y(and)189 350 y Fw(k)13 b FC(=)274 251 y Fg(8)274 289 y(>)274 301 y(<)274 376 y(>)274 388 y(:)332 293 y Fw(i)194 b FC(if)15 b Fw(u)e(>)g(v)332 350 y FC(min)q(\()p Fw(i;)8 b(j)s FC(\))39 b(if)15 b Fw(u)e FC(=)g Fw(v)332 406 y(j)191 b FC(if)15 b Fw(u)e(<)g(v)166 491 y FC(Note)21 b(that)g(ties)h(are)f (resolv)o(ed)h(in)h(fa)o(v)o(or)d(of)h(the)h(pro)q(cess)g(with)g(lo)o (w)o(er)f(rank)g(and)h(hence)h(this)75 548 y(op)q(eration)15 b(is)h(asso)q(ciativ)o(e)g(and)f(comm)o(utativ)o(e.)166 604 y Fl(MPI)p 243 604 13 2 v 14 w(MINLOC)g FC(is)h(de\014ned)g (similarly:)189 658 y Fg( )242 701 y Fw(u)247 758 y(i)289 658 y Fg(!)332 730 y FA(\016)365 658 y Fg( )418 701 y Fw(v)420 758 y(j)463 658 y Fg(!)508 730 y FC(=)556 658 y Fg( )610 701 y Fw(w)614 758 y(k)664 658 y Fg(!)75 856 y FC(where)189 942 y Fw(w)d FC(=)g(min)q(\()p Fw(u;)8 b(v)r FC(\))75 1028 y(and)189 1160 y Fw(k)13 b FC(=)274 1062 y Fg(8)274 1099 y(>)274 1112 y(<)274 1186 y(>)274 1199 y(:)332 1104 y Fw(i)194 b FC(if)15 b Fw(u)e(<)g(v)332 1160 y FC(min)q(\()p Fw(i;)8 b(j)s FC(\))39 b(if)15 b Fw(u)e FC(=)g Fw(v)332 1217 y(j)191 b FC(if)15 b Fw(u)e(>)g(v)75 1337 y Fv(Op)q(eration)j(/)f(T)l(yp)q(e)i(Compatibilit)o(y)75 1422 y FC(All)h(of)d(the)i(options)f(for)g Fv(op)g FC(in)h Fv(MPI)p 715 1422 14 2 v 16 w(REDUCE)g FC(do)f(not)g(apply)h(for)f(eac) o(h)g(of)g(the)g(p)q(ossible)i(MPI)e(basic)75 1479 y(datat)o(yp)q(es.)j (W)l(e)14 b(en)o(umerate)f(the)h(allo)o(w)o(ed)g(com)o(binations)h (here.)20 b(First,)13 b(de\014ne)i(groups)e(of)h(MPI)f(basic)75 1535 y(datat)o(yp)q(es)i(in)h(the)f(follo)o(wing)h(w)o(a)o(y)l(.)75 1631 y Fx(C)h(in)o(teger:)22 b Fo(MPI)p 400 1631 15 2 v 17 w(CHAR,)h(MPI)p 632 1631 V 17 w(INT,)g(MPI)p 840 1631 V 17 w(LONG,)g(MPI)p 1072 1631 V 17 w(SHORT,)189 1687 y(MPI)p 264 1687 V 16 w(UNSIGNED)p 472 1687 V 16 w(SHORT,)h(MPI)p 728 1687 V 16 w(UNSIGNED,)f(MPI)p 1055 1687 V 17 w(UNSIGNED)p 1264 1687 V 16 w(LONG)75 1778 y Fx(F)l(ortran)17 b(in)o(teger:)22 b Fo(MPI)p 535 1778 V 17 w(INTEGER)75 1869 y Fx(Floating)d(p)q(oin)o(t:)24 b Fo(MPI)p 513 1869 V 16 w(FLOAT,)g(MPI)p 769 1869 V 16 w(DOUBLE,)f(MPI)p 1048 1869 V 17 w(REAL,)189 1925 y(MPI)p 264 1925 V 16 w(DOUBLE)p 424 1925 V 17 w(PRECISION)75 2016 y Fx(Logical:)i Fo(MPI)p 349 2016 V 16 w(LOGICAL)75 2107 y Fx(Complex:)d Fo(MPI)p 390 2107 V 17 w(COMPLEX)75 2198 y Fx(Byte:)g Fo(MPI)p 296 2198 V 17 w(BYTE)166 2293 y FC(No)o(w,)14 b(the)h(v)m(alid)i(datat)o(yp)q(es)e(for)f(eac)o(h)i (option)f(is)h(sp)q(eci\014ed)h(b)q(elo)o(w.)117 2453 y Fl(MPI)p 194 2453 13 2 v 14 w(MAX,)c(MPI)p 400 2453 V 14 w(MIN,)h(MPI)p 592 2453 V 14 w(MAXLOC,)f(MPI)p 878 2453 V 14 w(MINLOC)41 b FC(C)10 b(in)o(teger,)h(F)l(ortran)e(in)o (teger,)i(Floating)f(p)q(oin)o(t)117 2506 y Fl(MPI)p 194 2506 V 14 w(SUM,)j(MPI)p 396 2506 V 14 w(PROD)424 b FC(C)10 b(in)o(teger,)h(F)l(ortran)e(in)o(teger,)i(Floating)f(p)q (oin)o(t,)h(Com-)905 2563 y(plex)117 2615 y Fl(MPI)p 194 2615 V 14 w(LAND,)j(MPI)p 418 2615 V 14 w(LOR,)f(MPI)p 612 2615 V 15 w(LX)o(OR)215 b FC(C)15 b(in)o(teger,)g(Logical)117 2668 y Fl(MPI)p 194 2668 V 14 w(BAND,)e(MPI)p 423 2668 V 14 w(BOR,)g(MPI)p 623 2668 V 14 w(BX)o(OR)199 b FC(C)15 b(in)o(teger,)g(Byte)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 93 101 93 100 bop 75 -100 a Fs(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FC(93)75 45 y Fv(Example)15 b(of)g(Reduce)75 131 y FC(Eac)o(h)h(pro)q(cess)h(has)g(an)f(arra)o(y)g(of)g(30)g Fo(double)p FC(s,)g(in)h(C.)f(F)l(or)g(eac)o(h)h(of)f(the)h(30)f(lo)q (cations,)h(compute)g(the)75 188 y(v)m(alue)f(and)g(rank)f(of)g(the)g (pro)q(cess)g(con)o(taining)h(the)f(largest)g(v)m(alue.)170 295 y Fo(...)170 351 y(/*)24 b(each)f(process)g(has)h(an)f(array)g(of)h (30)g(double:)f(a[])194 408 y(*/)170 464 y(double)g(a[30];)170 521 y(double)g(in[60],out[60];)170 577 y(int)h(i,outranks[30];)170 690 y(for)g(\(i=0;)f(i<30;)g(++i\))h({)266 747 y(in[i])f(=)h(a[i];)170 803 y(})170 859 y(MPI_Reduce\()f(in,)g(out,)h(30,)f(MPI_DOUBLE,)f (MPI_MAXLOC,)h(root,)g(comm)g(\);)170 916 y(/*)h(At)g(this)f(point,)g (the)g(answer)g(resides)g(on)h(process)f(root)194 972 y(*/)170 1029 y(if)h(\(myrank)f(==)h(root\))f({)266 1085 y(/*)g(read)h(ranks)f(out)g(and)h(coerce)f(back)g(to)h(ints)290 1142 y(*/)266 1198 y(for)f(\(i=0;)h(i<30;)f(++i\))g({)361 1255 y(outranks[i])g(=)g(out[30+i];)266 1311 y(})170 1368 y(})75 1490 y Fp(4.5.2)49 b(User-Reduce)75 1624 y Fv(MPI)p 160 1624 14 2 v 16 w(USER)p 288 1624 V 17 w(REDUCE\()16 b(sendbuf,)h(recvbuf,)f(count,)g(datat)o(yp)q(e,)g (function,)g(ro)q(ot,)f(comm\))117 1701 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c) o(hoice\))117 1776 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)19 b(address)h(of)e(receiv)o(e)i(bu\013er)f(\(c)o(hoice,)h(signi\014can)o (t)905 1833 y(only)13 b(at)h(ro)q(ot\))117 1908 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input) g(bu\013er)g(\(in)o(teger\))117 1984 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 2059 y(IN)171 b Fv(function)418 b Fq(user)15 b(de\014ned)g(function)f(\(function\))117 2135 y(IN)171 b Fv(ro)q(ot)492 b Fq(rank)14 b(of)f(ro)q(ot)h(pro)q(cess)i(\(in)o (teger\))117 2210 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 2335 y Fo(int)23 b(MPI)p 245 2335 15 2 v 17 w(User)p 358 2335 V 17 w(reduce\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)g(count,)393 2391 y(MPI)p 468 2391 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(int)i(root,)f(MPI)p 1607 2391 V 17 w(Comm)g(comm\))75 2478 y(MPI)p 150 2478 V 17 w(USER)p 263 2478 V 16 w(REDUCE\(SENDBUF,)f(RECVBUF,)h(COUNT,)g (DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM,)393 2534 y(IERROR\))170 2591 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2647 y(EXTERNAL)g(FUNCTION)170 2704 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g (COMM,)g(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 94 102 94 101 bop 75 -100 a FC(94)708 b Fs(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y FC(Similar)h(to)d(the)i (reduce)g(op)q(eration)f(function)h(ab)q(o)o(v)o(e)f(except)h(that)e(a) h(user)h(supplied)h(function)f(is)75 102 y(used.)39 b Fv(function)15 b FC(is)f(a)e(function)i(with)g(three)f(argumen)o(ts.)18 b(A)c(C)f(protot)o(yp)q(e)f(for)g(suc)o(h)i(function)g(is)g Fo(void)75 158 y(f\()24 b(invec,)f(inoutvec,)f(*len\))p FC(.)d(Both)28 b Fv(invec)15 b FC(and)29 b Fv(inoutvec)16 b FC(are)e(arra)o(ys)e(with)29 b Fv(*len)15 b FC(en)o(tries.)20 b(The)75 214 y(function)f(computes)f(elemen)o(t-wise)h(a)f(comm)o (utativ)o(e)f(and)h(asso)q(ciativ)o(e)g(op)q(eration)g(on)g(eac)o(h)g (pair)g(of)75 271 y(en)o(tries)13 b(and)g(returns)g(the)f(result)i(in) 26 b Fv(inoutvec)p FC(.)20 b(A)13 b(pseudo-co)q(de)h(for)25 b Fv(function)14 b FC(is)f(giv)o(en)g(b)q(elo)o(w,)h(where)75 327 y Fo(op)h FC(is)h(the)f(comm)o(utativ)o(e)f(and)i(asso)q(ciativ)o (e)f(op)q(eration)h(de\014ned)g(b)o(y)30 b Fv(function)p FC(.)361 430 y Fo(for\(i=0;)23 b(i)h(<)g(*len;)f(i++\))g({)552 486 y(inoutvec[i])g(op=)g(invec[i])361 543 y(})166 646 y FC(The)16 b(t)o(yp)q(e)h(of)f(the)g(elemen)o(ts)h(of)32 b Fv(invec)17 b FC(and)g(of)32 b Fv(inoutvec)18 b FC(matc)o(h)d(the)i (t)o(yp)q(e)f(of)g(the)g(elemen)o(ts)h(of)75 702 y(the)i(send)g (bu\013ers)f(and)h(the)f(receiv)o(e)i(bu\013er.)30 b(No)18 b(MPI)g(functions)i(ma)o(y)e(b)q(e)h(called)h(inside)g(the)f(user)75 758 y(de\014ned)e(function.)166 815 y(The)k(addition)i(of)d(the)h (third)h(argumen)o(t,)44 b Fv(*len)p FC(,)23 b(in)43 b Fv(function)22 b FC(allo)o(ws)g(the)f(system)g(to)f(a)o(v)o(oid)75 871 y(calling)40 b Fv(function)21 b FC(for)d(eac)o(h)i(elemen)o(t)g(in) g(the)f(input)h(bu\013er.)32 b(Rather,)20 b(the)f(system)g(can)g(c)o (ho)q(ose)h(to)75 928 y(apply)31 b Fv(function)17 b FC(to)d(c)o(h)o (unks)i(of)f(input.)75 1047 y Fv(Example)g(of)g(User-Reduce)75 1133 y FC(Compute)g(the)g(pro)q(duct)h(of)f(an)g(arra)o(y)f(of)g (complex)i(n)o(um)o(b)q(ers,)g(in)g(C.)75 1236 y Fo(typedef)23 b(struct)g(_complex)g({)170 1292 y(double)g(real,imag;)75 1349 y(})h(Complex;)75 1462 y(/*)g(the)f(user-defined)f(function)99 1518 y(*/)75 1631 y(void)h(myProd\()g(Complex)g(invec[],)g(Complex)g (inoutvec[],)f(int)i(*len)f(\))75 1687 y({)170 1744 y(int)h(i;)170 1800 y(Complex)f(c;)170 1913 y(for)h(\(i=0;)f(i<)h(*len;)f(++i\))g({) 266 1970 y(c.real)g(=)h(inoutvec[i].real*invec[i)o(].real)c(-)528 2026 y(inoutvec[i].imag*invec[i].ima)o(g;)266 2083 y(c.imag)j(=)h (inoutvec[i].real*invec[i)o(].imag)c(+)528 2139 y (inoutvec[i].imag*invec[i].rea)o(l;)266 2195 y(inoutvec[i])i(=)i(c;)170 2252 y(})75 2308 y(})75 2421 y(/*)g(and,)f(to)g(call)h(it...)99 2478 y(*/)75 2534 y(...)170 2647 y(/*)g(each)f(process)g(has)h(an)f (array)g(of)h(100)g(Complexes)194 2704 y(*/)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 95 103 95 102 bop 75 -100 a Fs(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FC(95)170 45 y Fo(Complex)23 b(a[100],)g(answer[100];)170 102 y(MPI_Datatype)g(ctype;)170 214 y(/*)h(explain)f(to)h(MPI)f(what)g (a)h("complex")f(is)g(\(remember,)g(this)g(is)h(C!\))194 271 y(*)g(Note)f(we)h(are)f(assuming)g(compiler)g(implements)f(type)i (Complex)f(as)194 327 y(*)h(two)f(contiguous)g(doubles)194 384 y(*/)170 440 y(MPI_Type_contiguous\()f(2,)h(MPI_DOUBLE,)g(&ctype)g (\);)170 497 y(MPI_Type_commit\()f(ctype)h(\);)170 610 y(MPI_User_reduce\()f(a,)i(answer,)f(100,)g(ctype,)g(myProd,)g(root,)g (comm)g(\);)170 723 y(/*)h(At)g(this)f(point,)g(the)g(answer,)g(which)h (consists)e(of)i(100)f(Complexes,)194 779 y(*)h(resides)f(on)h(process) e(root)194 835 y(*/)75 999 y Fv(MPI)p 160 999 14 2 v 16 w(USER)p 288 999 V 17 w(REDUCEA\()16 b(sendbuf,)h(recvbuf,)f(count,) g(datat)o(yp)q(e,)g(function,)h(ro)q(ot,)d(comm\))117 1078 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 1157 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)19 b(address)h(of)e(receiv)o(e)i(bu\013er) f(\(c)o(hoice,)h(signi\014can)o(t)905 1213 y(only)13 b(at)h(ro)q(ot\))117 1292 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er) 13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g(\(in)o(teger\))117 1372 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(input)h(bu\013er)h(\(handle\))117 1451 y(IN)171 b Fv(function)418 b Fq(user)15 b(de\014ned)g(function)f (\(function\))117 1530 y(IN)171 b Fv(ro)q(ot)492 b Fq(rank)14 b(of)f(ro)q(ot)h(pro)q(cess)i(\(in)o(teger\))117 1609 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 1735 y Fo(int)23 b(MPI)p 245 1735 15 2 v 17 w(User)p 358 1735 V 17 w(reducea\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g (count,)393 1792 y(MPI)p 468 1792 V 17 w(Datatype)g(datatype,)f(void)i (function,)e(int)i(root,)f(MPI)p 1607 1792 V 17 w(Comm)g(comm\))75 1880 y(MPI)p 150 1880 V 17 w(USER)p 263 1880 V 16 w(REDUCEA\(SENDBUF,)f (RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM,)393 1936 y(IERROR\))170 1993 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2049 y(EXTERNAL)g(FUNCTION)170 2106 y(INTEGER)g(COUNT,)g(DATATYPE,)g (ROOT,)g(COMM,)g(IERROR)166 2194 y FC(Iden)o(tical)16 b(to)f Fv(MPI)p 493 2194 14 2 v 16 w(USER)p 621 2194 V 17 w(REDUCE)p FC(,)g(except)g(that)g(the)g(op)q(eration)g(de\014ned)h (b)o(y)30 b Fv(function)16 b FC(is)f(not)75 2251 y(required)22 b(to)e(b)q(e)i(comm)o(utativ)o(e,)f(but)g(only)h(asso)q(ciativ)o(e.)37 b(Th)o(us,)22 b(the)f(order)f(of)h(computation)g(can)75 2307 y(b)q(e)d(mo)q(di\014ed)h(only)e(using)h(asso)q(ciativit)o(y)l(.) 27 b(Use)17 b(of)g(this)h(function)g(means)f(that)f(the)i(implemen)o (tation)75 2364 y Fr(c)n(annot)c FC(try)g(and)h(c)o(hange)f(the)h (order)f(of)g(computation)h(of)f(the)h(reduce)g(b)o(y)g(assuming)g (comm)o(utativit)o(y)l(.)166 2498 y Fn(Implemen)o(tati)o(on)d(note:)166 2556 y Fl(MPI)p 243 2556 13 2 v 14 w(USER)p 361 2556 V 14 w(REDUCEA)e Fq(and)h Fl(MPI)p 729 2556 V 14 w(USER)p 847 2556 V 15 w(REDUCE)e Fq(can)j(ha)o(v)o(e)f(iden)o(tical)g(implemen) o(tati)o(ons,)e(if)i(one)g(do)q(es)75 2613 y(not)j(wish)g(to)f(tak)o(e) h(adv)n(an)o(tage)f(of)h(comm)o(utati)o(vit)o(y)l(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 96 104 96 103 bop 75 -100 a FC(96)703 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fp(4.5.3)49 b(All-Reduce)75 142 y FC(MPI)20 b(includes)i(v)m(arian)o(ts)e(of)f(eac) o(h)h(of)g(the)g(reduce)g(op)q(erations)g(where)g(the)g(result)h(is)f (kno)o(wn)g(to)f(all)75 199 y(pro)q(cesses)f(in)h(the)f(group)g(on)g (return.)28 b(MPI)18 b(requires)g(that)g(all)h(pro)q(cesses)f (participating)h(in)g(an)o(y)e(of)75 255 y(the)e(all-reduce)i(op)q (erations)f(receiv)o(e)g(exactly)g(iden)o(tical)h(results.)75 365 y Fv(MPI)p 160 365 14 2 v 16 w(ALLREDUCE\()e(sendbuf,)i(recvbuf,)f (count,)g(datat)o(yp)q(e,)g(op,)f(comm\))117 448 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 535 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 622 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 708 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h (of)f(send)i(bu\013er)g(\(handle\))117 795 y(IN)171 b Fv(op)525 b Fq(op)q(eration)14 b(\(state\))117 882 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 1012 y Fo(int)23 b(MPI)p 245 1012 15 2 v 17 w(Allreduce\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 1069 y(MPI)p 468 1069 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1069 V 17 w(Op)i(op,)f(MPI)p 1266 1069 V 17 w(Comm)g(comm\))75 1161 y(MPI)p 150 1161 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g (DATATYPE,)g(OP,)g(COMM,)g(IERROR\))170 1218 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1274 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1366 y FC(Same)19 b(as)h(the)f Fv(MPI)p 519 1366 14 2 v 16 w(REDUCE)h FC(op)q(eration)g(function)g(except)g(that)f (the)h(result)g(app)q(ears)f(in)i(the)75 1423 y(receiv)o(e)16 b(bu\013er)f(of)g(all)h(the)f(group)g(mem)o(b)q(ers.)75 1533 y Fv(MPI)p 160 1533 V 16 w(USER)p 288 1533 V 17 w(ALLREDUCE\()g(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h (function,)f(comm\))117 1616 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1702 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1789 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 1876 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(send)i(bu\013er)g (\(handle\))117 1963 y(IN)171 b Fv(function)418 b Fq(user)15 b(de\014ned)g(function)f(\(function\))117 2050 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 2180 y Fo(int)23 b(MPI)p 245 2180 15 2 v 17 w(User)p 358 2180 V 17 w(allreduce\(void*)f(sendbuf,)g(void*)i(recvbuf,)e(int)i (count,)393 2237 y(MPI)p 468 2237 V 17 w(Datatype)f(datatype,)f(void)i (function,)e(MPI)p 1368 2237 V 17 w(Comm)h(comm\))75 2329 y(MPI)p 150 2329 V 17 w(USER)p 263 2329 V 16 w (ALLREDUCE\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)f(FUNCTION,)h (COMM,)393 2385 y(IERROR\))170 2442 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2498 y(EXTERNAL)g(FUNCTION)170 2555 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)166 2647 y FC(Same)15 b(as)g(the)g Fv(MPI)p 506 2647 14 2 v 16 w(USER)p 634 2647 V 18 w(REDUCE)h FC(op)q(eration)f(function)h(except)g (that)f(the)g(result)h(app)q(ears)f(in)75 2704 y(the)g(receiv)o(e)h (bu\013er)g(of)e(all)i(the)g(group)f(mem)o(b)q(ers.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 97 105 97 104 bop 75 -100 a Fs(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FC(97)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(USER)p 288 45 V 17 w(ALLREDUCEA\()16 b(sendbuf,)h(recvbuf,)e(count,)h(datat)o (yp)q(e,)h(function,)f(comm\))117 123 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 200 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 277 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h (bu\013er)f(\(in)o(teger\))117 354 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(send)i(bu\013er)g (\(handle\))117 431 y(IN)171 b Fv(function)418 b Fq(user)15 b(de\014ned)g(function)f(\(function\))117 508 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 634 y Fo(int)23 b(MPI)p 245 634 15 2 v 17 w(User)p 358 634 V 17 w(allreducea\(void*)f(sendbuf,)g(void*)h(recvbuf,)g(int)h (count,)393 690 y(MPI)p 468 690 V 17 w(Datatype)f(datatype,)f(void)i (function,)e(MPI)p 1368 690 V 17 w(Comm)h(comm\))75 777 y(MPI)p 150 777 V 17 w(USER)p 263 777 V 16 w(ALLREDUCEA\(SENDBUF,)f (RECVBUF,)g(COUNT,)h(DATATYPE,)g(FUNCTION,)g(COMM,)393 834 y(IERROR\))170 890 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 947 y(EXTERNAL)g(FUNCTION)170 1003 y(INTEGER)g(COUNT,)g(DATATYPE,)g (COMM,)g(IERROR)166 1091 y FC(Same)16 b(as)f Fv(MPI)p 429 1091 14 2 v 16 w(USER)p 557 1091 V 18 w(REDUCEA)p FC(,)h(except)g(that)g(the)g(result)g(app)q(ears)g(in)h(the)f(receiv)o (e)h(bu\013er)e(of)75 1147 y(all)h(the)f(group)g(mem)o(b)q(ers.)166 1287 y Fn(Implemen)o(tati)o(on)c(note:)36 b Fq(The)13 b(all-reduce)h(op)q(erations)f(can)g(b)q(e)h(implemen)o(ted)d(as)i(a)g (reduce,)i(follo)o(w)o(ed)75 1344 y(b)o(y)f(a)f(broadcast.)19 b(Ho)o(w)o(ev)o(er,)14 b(a)f(direct)i(implemen)o(tatio)o(n)c(can)j (lead)g(to)g(b)q(etter)h(p)q(erformance.)75 1553 y Fp(4.5.4)49 b(Reduce-Scatter)75 1641 y FC(MPI)15 b(also)f(includes)j(v)m(arian)o (ts)e(of)f(eac)o(h)g(of)h(the)f(reduce)i(op)q(erations)f(where)f(the)h (result)g(is)g(scattered)g(to)75 1697 y(all)h(pro)q(cesses)g(in)g(the)f (group)g(on)g(return.)75 1802 y Fv(MPI)p 160 1802 V 16 w(REDUCE)p 352 1802 V 17 w(SCA)l(TTER\()h(sendbuf,)h(recvbuf,)f (recvcounts,)g(datat)o(yp)q(e,)h(op,)e(comm\))117 1880 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1957 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er)g(\(c)o (hoice\))117 2034 y(IN)171 b Fv(recvcounts)372 b Fq(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g(the)h(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g (in)f(re-)905 2091 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 2147 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 2224 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 2301 y(IN)171 b Fv(op)525 b Fq(op)q(eration)14 b(\(state\))117 2378 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 2503 y Fo(int)23 b(MPI)p 245 2503 15 2 v 17 w(Reduce)p 406 2503 V 17 w(scatter\(void*)f(sendbuf,)g(void*)i (recvbuf,)e(int)i(*recvcounts,)393 2560 y(MPI)p 468 2560 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2560 V 17 w(Op)i(op,)f(MPI)p 1266 2560 V 17 w(Comm)g(comm\))75 2647 y(MPI)p 150 2647 V 17 w(REDUCE)p 311 2647 V 16 w(SCATTER\(SENDBUF,)f(RECVBUF,)h (RECVCOUNTS,)f(DATATYPE,)h(OP,)g(COMM,)393 2704 y(IERROR\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 98 106 98 105 bop 75 -100 a FC(98)703 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 45 y Fo()23 b(SENDBUF\(*\),)g(RECVBUF\(*\))170 102 y(INTEGER)g(RECVCOUNTS,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR)166 199 y Fv(MPI)p 251 199 14 2 v 16 w(REDUCE)p 443 199 V 17 w(SCA)l(TTER)16 b FC(\014rst)f(do)q(es)h(a)f(comp)q(onen)o(t)o(wise)g(reduction)h(on)g (v)o(ectors)e(pro)o(vided)i(b)o(y)75 256 y(the)h(pro)q(cesses.)24 b(Next,)17 b(the)f(resulting)i(v)o(ector)e(of)g(results)h(is)g(split)g (in)o(to)g Fo(n)f FC(disjoin)o(t)i(segmen)o(ts,)e(where)75 312 y Fo(n)h FC(is)g(the)g(n)o(um)o(b)q(er)g(of)f(mem)o(b)q(ers)h(in)h (the)f(group;)g(segmen)o(t)f Fo(i)h FC(con)o(tains)g Fv(recvcounts[i])h FC(elemen)o(ts.)26 b(The)75 368 y Fo(i)p FC(-th)15 b(segmen)o(t)g(is)h(sen)o(t)f(to)f(pro)q(cess)i(with)f (rank)g Fo(i)p FC(.)166 519 y Fn(Implemen)o(tati)o(on)10 b(note:)17 b Fq(The)12 b Fl(MPI)p 792 519 13 2 v 15 w(REDUCE)p 970 519 V 13 w(SCA)m(TTER)f Fq(routine)h(is)g(functionally)f(equiv)n (alen)o(t)g(to:)17 b(A)75 575 y Fl(MPI)p 152 575 V 14 w(REDUCE)28 b Fq(op)q(eration)g(function)h(with)f Fl(count)i Fq(equal)f(to)f(the)i(sum)e(of)g Fl(recvcounts[i])j Fq(follo)o(w)o(ed)c (b)o(y)75 632 y Fl(MPI)p 152 632 V 14 w(SCA)m(TTERV)15 b Fq(with)i Fl(sendcounts)h Fq(equal)f(to)f Fl(recvcounts)p Fq(.)28 b(Ho)o(w)o(ev)o(er,)17 b(a)g(direct)g(implemen)o(tatio)o(n)d (ma)o(y)h(run)75 688 y(faster.)75 886 y Fv(MPI)p 160 886 14 2 v 16 w(USER)p 288 886 V 17 w(REDUCE)p 481 886 V 17 w(SCA)l(TTER\()h(sendbuf,)i(recvbuf,)d(recvcnts,)h(t)o(yp)q(e,)g (function,)g(comm\))117 974 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1071 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1168 y(IN)171 b Fv(recvcnts)418 b Fq(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g(the)h (n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(in)f(re-)905 1225 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 1281 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 1379 y(IN)171 b Fv(t)o(yp)q(e)491 b Fq(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 1476 y(IN)171 b Fv(function)418 b Fq(user)15 b(de\014ned)g(function)f(\(function\))117 1573 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))166 1709 y FC(Same)j(as)g(the)g Fv(MPI)p 503 1709 V 16 w(REDUCE)p 695 1709 V 17 w(SCA)l(TTER)i FC(op)q(eration)e(function)h(except)g (that)e(the)i(user)f(sp)q(eci\014es)75 1765 y(the)h(reduction)h(op)q (eration)g(as)f(in)h Fv(MPI)p 753 1765 V 15 w(USER)p 880 1765 V 18 w(REDUCE)p FC(.)75 1880 y Fv(MPI)p 160 1880 V 16 w(USER)p 288 1880 V 17 w(REDUCE)p 481 1880 V 17 w(SCA)l(TTERA\()h(sendbuf,)g(recvbuf,)e(recvcnts,)i(t)o(yp)q(e,)e (function,)i(comm\))117 1968 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 2066 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 2163 y(IN)171 b Fv(recvcnts)418 b Fq(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g(the)h (n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(in)f(re-)905 2220 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 2276 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 2373 y(IN)171 b Fv(t)o(yp)q(e)491 b Fq(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 2471 y(IN)171 b Fv(function)418 b Fq(user)15 b(de\014ned)g(function)f(\(function\))117 2568 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))166 2704 y FC(The)k(\\A")g(v)o(ersion)g(of)g Fv(MPI)p 645 2704 V 16 w(USER)p 773 2704 V 17 w(REDUCE)p 966 2704 V 18 w(SCA)l(TTER)p FC(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 99 107 99 106 bop 75 -100 a Fs(4.5.)34 b(GLOBAL)16 b(COMPUTE)f(OPERA)l(TIONS) 883 b FC(99)75 45 y Fp(4.5.5)49 b(Scan)75 183 y Fv(MPI)p 160 183 14 2 v 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat)o (yp)q(e,)h(op,)e(comm)e(\))117 262 y Fq(IN)171 b Fv(sendbuf)428 b Fq(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 342 y(OUT)124 b Fv(recvbuf)434 b Fq(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 422 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input) g(bu\013er)g(\(in)o(teger\))117 502 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 582 y(IN)171 b Fv(op)525 b Fq(op)q(eration)14 b(\(state\))117 662 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 789 y Fo(int)23 b(MPI)p 245 789 15 2 v 17 w(Scan\(void*)g(sendbuf,)f(void*)i(recvbuf,)e(int)i(count,)393 845 y(MPI)p 468 845 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 845 V 17 w(Op)i(op,)f(MPI)p 1266 845 V 17 w(Comm)g(comm)h(\))75 934 y(MPI)p 150 934 V 17 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 991 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1047 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1136 y Fv(MPI)p 251 1136 14 2 v 16 w(SCAN)f FC(is)g(used)h(to)e(p)q(erform)g(a)h(parallel)h(pre\014x)f(with)g(resp) q(ect)g(to)f(an)h(asso)q(ciativ)o(e)g(and)75 1193 y(comm)o(utativ)o(e)e (reduction)i(op)q(eration)g(on)f(data)f(distributed)j(across)d(the)h (group.)40 b(The)23 b(op)q(eration)75 1249 y(returns)17 b(in)h(the)f(receiv)o(e)i(bu\013er)e(of)g(the)g(pro)q(cess)g(with)h (rank)f Fo(i)g FC(the)g(reduction)h(of)f(the)g(v)m(alues)i(in)f(the)75 1305 y(send)k(bu\013ers)f(of)f(pro)q(cesses)i(with)g(ranks)e Fo(0,...,i)p FC(.)37 b(The)21 b(t)o(yp)q(e)h(of)e(op)q(erations)i(supp) q(orted,)g(their)75 1362 y(seman)o(tics,)15 b(and)g(the)h(constrain)o (ts)e(on)h(send)h(and)g(receiv)o(e)g(bu\013ers)f(are)g(as)g(for)f Fv(MPI)p 1531 1362 V 16 w(REDUCE)p FC(.)75 1468 y Fv(MPI)p 160 1468 V 16 w(USER)p 288 1468 V 17 w(SCAN\()i(sendbuf,)h(recvbuf,)f (count,)g(datat)o(yp)q(e,)g(function,)g(comm\))117 1548 y Fq(IN)171 b Fv(sendbuf)428 b Fq(address)15 b(of)f(input)f(bu\013er) 117 1628 y(OUT)124 b Fv(recvbuf)434 b Fq(address)15 b(of)f(output)g (bu\013er)117 1708 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input)h(and)g(output)g(bu\013er)h(\(in)o(te-) 905 1764 y(ger\))117 1844 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f(bu\013er)i(\(handle\))117 1924 y(IN)171 b Fv(function)418 b Fq(user)15 b(pro)o(vided)f(function) 117 2004 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 2131 y Fo(int)23 b(MPI)p 245 2131 15 2 v 17 w(User)p 358 2131 V 17 w(scan\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)h(count,)393 2187 y(MPI)p 468 2187 V 17 w(Datatype)f(datatype,)f(void)i(function,)e(MPI)p 1368 2187 V 17 w(Comm)h(comm\))75 2276 y(MPI)p 150 2276 V 17 w(USER)p 263 2276 V 16 w(SCAN\(SENDBUF,)f(RECVBUF,)h(COUNT,)g (DATATYPE,)g(FUNCTION,)g(COMM,)g(IERROR\))170 2332 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 2389 y(EXTERNAL)g(FUNCTION)170 2445 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)166 2534 y FC(Same)12 b(as)g(the)h Fv(MPI)p 498 2534 14 2 v 15 w(SCAN)g FC(op)q(eration)g(function)g(except)g(that)f(a)g(user)g (supplied)j(function)e(is)g(used.)75 2591 y Fv(function)k FC(is)f(an)g(asso)q(ciativ)o(e)f(and)h(comm)o(utativ)o(e)f(function)h (with)g(an)g(input)g(v)o(ector,)f(an)g(inout)h(v)o(ector,)75 2647 y(and)g(a)g(length)g(argumen)o(t.)21 b(The)16 b(t)o(yp)q(es)g(of)g (the)g(t)o(w)o(o)e(v)o(ectors)h(and)h(of)g(the)g(returned)g(v)m(alues)h (all)g(agree.)75 2704 y(See)f Fv(MPI)p 241 2704 V 16 w(USER)p 369 2704 V 17 w(REDUCE)g FC(for)e(more)h(details.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 100 108 100 107 bop 75 -100 a FC(100)680 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(USER)p 288 45 V 17 w(SCANA\()g(sendbuf,)h(recvbuf,)f (count,)g(datat)o(yp)q(e,)g(function,)h(comm\))117 123 y Fq(IN)171 b Fv(sendbuf)428 b Fq(address)15 b(of)f(input)f(bu\013er)i (\(c)o(hoice\))117 200 y(OUT)124 b Fv(recvbuf)434 b Fq(address)15 b(of)f(output)g(bu\013er)h(\(c)o(hoice\))117 277 y(IN)171 b Fv(count)466 b Fq(n)o(um)o(b)q(er)14 b(of)h(elemen)o(ts)g(in)f(input) h(and)g(output)g(bu\013er)h(\(in)o(te-)905 333 y(ger\))117 410 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fq(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)i(\(handle\))117 486 y(IN)171 b Fv(function)418 b Fq(user)15 b(de\014ned)g(function)f(\(function\))117 563 y(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))75 688 y Fo(int)23 b(MPI)p 245 688 15 2 v 17 w(User)p 358 688 V 17 w(scana\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g(count,) 393 745 y(MPI)p 468 745 V 17 w(Datatype)g(datatype,)f(void)i(function,) e(MPI)p 1368 745 V 17 w(Comm)h(comm\))75 832 y(MPI)p 150 832 V 17 w(USER)p 263 832 V 16 w(SCANA\(SENDBUF,)f(RECVBUF,)h (COUNT,)g(DATATYPE,)g(FUNCTION,)f(COMM,)i(IERROR\))170 889 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 945 y(EXTERNAL)g (FUNCTION)170 1002 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)166 1089 y FC(Same)13 b(as)g Fv(MPI)p 424 1089 14 2 v 16 w(USER)p 552 1089 V 17 w(SCAN)p FC(,)h(except)f(that)g(the)g (user-de\014ned)i(op)q(eration)e(need)h(not)f(b)q(e)h(comm)o(u-)75 1145 y(tativ)o(e.)166 1278 y Fn(Implemen)o(tati)o(on)e(note:)166 1329 y Fl(MPI)p 243 1329 13 2 v 14 w(USER)p 361 1329 V 14 w(SCAN)j Fq(could)g(b)q(e)g(implemen)o(ted)e(as)i Fl(MPI)p 1040 1329 V 14 w(USER)p 1158 1329 V 15 w(SCANA)p Fq(,)e(though)i(for)g(some)f(arc)o(hitectures)75 1379 y(it)g(ma)o(y)d(b)q(e)k(p)q(ossible)f(to)g(arriv)o(e)g(at)f(a)h(faster) g Fl(MPI)p 864 1379 V 15 w(USER)p 983 1379 V 14 w(SCAN)f Fq(b)o(y)h(taking)f(adv)n(an)o(tage)g(of)g(comm)o(utativit)o(y)l(.)75 1609 y Ft(4.6)59 b(Co)n(rrectness)75 1714 y Fp(4.6.1)49 b(Synchronization)18 b(Side-E\013ects)75 1801 y FC(A)13 b(correct)f(program)g(should)i(in)o(v)o(ok)o(e)f(collectiv)o(e)h(comm)o (unications)g(so)e(that)g(deadlo)q(c)o(k)i(will)g(not)f(o)q(ccur,)75 1858 y(whether)f(collectiv)o(e)h(comm)o(unication)g(is)f(sync)o (hronizing)h(or)f(not.)18 b(The)12 b(follo)o(wing)g(examples)h (illustrate)75 1914 y(dangerous)i(use)h(of)e(collectiv)o(e)j(routines.) 166 1972 y(The)e(follo)o(wing)h(example)g(illustrates)h(an)e(erroneous) g(program.)75 2083 y Fo(/*)24 b(Example)e(A)i(*/)75 2139 y(switch\(rank\))147 2195 y({)147 2252 y(case)f(0:)g({)h (MPI_Bcast\(var1,)e(count,)h(type,)g(0,)h(comm\);)385 2308 y(MPI_Send\(var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)385 2365 y(break;)337 2421 y(})147 2478 y(case)g(1:)g({)h(MPI_Recv\(var2,)e (count,)h(type,)g(0,)h(tag,)f(comm\);)385 2534 y(MPI_Bcast\(var1,)f (count,)h(type,)g(0,)h(comm\);)385 2591 y(break;)337 2647 y(})147 2704 y(})1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 101 109 101 108 bop 75 -100 a Fs(4.6.)34 b(CORRECTNESS)1280 b FC(101)166 45 y(Pro)q(cess)16 b(zero)f(executes)h(a)g(broadcast,)f (follo)o(w)o(ed)h(b)o(y)f(a)h(blo)q(c)o(king)h(send)f(op)q(eration;)g (pro)q(cess)g(one)75 102 y(\014rst)h(executes)h(a)g(matc)o(hing)f(blo)q (c)o(king)i(receiv)o(e,)g(follo)o(w)o(ed)f(b)o(y)g(the)f(matc)o(hing)h (broadcast)f(call.)28 b(This)75 158 y(program)18 b(ma)o(y)h(deadlo)q(c) o(k.)32 b(The)20 b(broadcast)e(call)j(on)e(pro)q(cess)g(zero)g Fr(may)h FC(blo)q(c)o(k)g(un)o(til)g(pro)q(cess)g(one)75 214 y(executes)f(the)g(matc)o(hing)f(broadcast)g(call,)i(so)e(that)f (the)i(send)g(is)g(not)f(executed.)30 b(Pro)q(cess)19 b(one)f(will)75 271 y(de\014nitely)f(blo)q(c)o(k)f(on)f(the)h(receiv)o (e)g(and)f(so,)g(in)h(this)f(case,)g(nev)o(er)g(executes)h(the)f (broadcast.)166 329 y(The)g(follo)o(wing)h(example)g(is)g(correct,)e (but)i(non-deterministic:)75 445 y Fo(/*)24 b(Example)e(B)i(*/)75 501 y(switch\(rank\))147 558 y({)170 614 y(case)g(0:)f({)h (MPI_Bcast\(var1,)e(count,)h(type,)g(0,)h(comm\);)409 670 y(MPI_Send\(var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)409 727 y(break;)361 783 y(})170 840 y(case)h(1:)f({)h(MPI_Recv\(var2,)e (count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h(comm\);)409 896 y(MPI_Bcast\(var1,)f(count,)h(type,)g(0,)h(comm\);)409 953 y(MPI_Recv\(var2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h (comm\);)409 1009 y(break;)361 1066 y(})170 1122 y(case)h(2:)f({)h (MPI_Send\(var2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)409 1179 y(MPI_Bcast\(var1,)f(count,)h(type,)g(0,)h(comm\);)409 1235 y(break;)361 1291 y(})170 1348 y(})166 1462 y FC(All)19 b(three)f(pro)q(cesses)g(participate)h(in)g(a)e(broadcast.)28 b(Pro)q(cess)17 b(0)h(sends)g(a)g(message)f(to)h(pro)q(cess)75 1518 y(1)h(after)g(the)h(broadcast,)f(and)h(pro)q(cess)f(2)h(sends)g(a) f(message)g(to)g(pro)q(cess)h(1)f(b)q(efore)g(the)h(broadcast.)75 1575 y(Pro)q(cess)15 b(1)g(receiv)o(es)h(b)q(efore)g(and)f(after)f(the) i(broadcast,)e(with)h(a)g(wildcard)i(source)e(argumen)o(t.)166 1633 y(Tw)o(o)e(p)q(ossible)i(executions,)g(with)e(di\013eren)o(t)h (matc)o(hings)g(of)f(sends)h(and)g(receiv)o(es,)g(are)g(illustrated)75 1689 y(in)i(\014gure)f(4.10.)166 1747 y(Note)h(that)g(the)h(second)g (execution)h(has)f(the)f(p)q(eculiar)j(e\013ect)e(that)f(a)g(send)h (executed)h(after)e(the)75 1804 y(broadcast)i(is)h(receiv)o(ed)g(at)f (another)g(no)q(de)h(b)q(efore)g(the)f(broadcast.)29 b(This)19 b(example)g(illustrates)h(the)75 1860 y(fact)i(that)f(one)i (should)g(not)f(rely)h(on)f(collectiv)o(e)i(comm)o(unication)g (functions)f(to)e(ha)o(v)o(e)h(particular)75 1917 y(sync)o(hronization) d(e\013ects.)29 b(T)l(o)17 b(assume)h(that)g(collectiv)o(e)i(comm)o (unication)f(functions)g(do)f(or)f(do)h(not)75 1973 y(ha)o(v)o(e)d (certain)g(sync)o(hronizing)i(side-e\013ects)f(is)g(non-p)q(ortable.)75 2104 y Fp(4.6.2)49 b(Multiple)17 b(Calls)h(to)e(Collective)75 2193 y FC(It)d(is)h(the)g(user's)f(resp)q(onsibilit)o(y)j(to)d(mak)o(e) f(sure)i(that)f(there)g(are)g(no)h(t)o(w)o(o)e(concurren)o(tly)i (executing)g(col-)75 2249 y(lectiv)o(e)h(calls)g(that)f(use)g(the)g (same)g(comm)o(unicator)g(on)g(the)g(same)g(pro)q(cess.)20 b(Since)15 b(all)g(collectiv)o(e)h(com-)75 2306 y(m)o(unication)j (calls)g(are)f(blo)q(c)o(king)h(this)g(restriction)f(only)h(a\013ects)e (m)o(ultithreaded)i(implemen)o(tations.)75 2362 y(On)c(the)g(other)f (hand,)h(it)g(is)g(legitimate)h(for)e(one)g(pro)q(cess)h(to)f(start)g (a)g(new)h(collectiv)o(e)h(comm)o(unication)75 2419 y(call)i(ev)o(en)e (though)h(a)f(previous)h(call)h(that)e(uses)g(the)h(same)f(comm)o (unicator)g(has)h(not)f(y)o(et)g(terminated)75 2475 y(on)f(another)g (pro)q(cess.)20 b(As)15 b(illustrated)i(in)f(the)f(follo)o(wing)h (example:)75 2591 y Fo(/*)24 b(Example)e(C)i(*/)99 2647 y(MPI_Bcast\(var1,)e(count,)h(type,)g(0,)g(comm\);)99 2704 y(MPI_Bcast\(var2,)f(count,)h(type,)g(1,)g(comm\);)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 102 110 102 109 bop 75 -100 a FC(102)680 b Fs(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)383 4 y 18945146 16482269 0 0 29733355 25918013 startTexFig 383 4 a %%BeginDocument: coll-matchings.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -50 0 translate -40.0 436.0 translate 0.900 -0.900 scale 0.500 setlinewidth n 469 134 m 334 134 l gs col-1 s gr n 342.000 136.000 m 334.000 134.000 l 342.000 132.000 l gs 2 setlinejoin col-1 s gr n 154 399 m 284 399 l gs col-1 s gr n 276.000 397.000 m 284.000 399.000 l 276.000 401.000 l gs 2 setlinejoin col-1 s gr n 469 454 m 329 454 l gs col-1 s gr n 337.000 456.000 m 329.000 454.000 l 337.000 452.000 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 279 59 m gs 1 -1 scale (First Execution) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 274 299 m gs 1 -1 scale (Second Execution) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 139 94 m gs 1 -1 scale (0) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 319 94 m gs 1 -1 scale (1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 509 94 m gs 1 -1 scale (2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 194 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 194 m gs 1 -1 scale (send) col-1 show gr n 159 189 m 279 189 l gs col-1 s gr n 271.000 187.000 m 279.000 189.000 l 271.000 191.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 294 139 m gs 1 -1 scale (recv) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 44 94 m gs 1 -1 scale (process:) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 139 m gs 1 -1 scale (send) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 199 184 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 384 129 m gs 1 -1 scale (match) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 429 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 404 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 459 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 459 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 484 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 404 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 379 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 194 394 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 379 449 m gs 1 -1 scale (match) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1146 a FC(Figure)d(4.10:)k(A)c(race)f(condition)i(causes)f (non-deterministic)i(matc)o(hing)d(of)g(sends)i(and)e(receiv)o(es.)20 b(One)75 1202 y(cannot)15 b(rely)h(on)f(sync)o(hronization)h(from)e(a)h (broadcast)g(to)f(mak)o(e)h(the)g(program)f(deterministic.)166 1334 y(In)j(a)e(nonsync)o(hronizing)j(implemen)o(tation)g(of)d (broadcast,)g(pro)q(cess)i(zero)f(ma)o(y)f(start)g(executing)75 1390 y(the)h(second)g(broadcast)f(b)q(efore)h(pro)q(cess)g(one)g (terminated)g(the)g(\014rst)f(broadcast.)21 b(Both)15 b(pro)q(cess)h(zero)75 1447 y(and)h(one)f(ma)o(y)g(terminate)g(their)h (t)o(w)o(o)e(broadcast)g(calls)j(b)q(efore)e(other)g(pro)q(cesses)h(ha) o(v)o(e)f(started)f(their)75 1503 y(calls.)21 b(It)15 b(is)h(the)f(implemen)o(tor's)h(resp)q(onsibilit)o(y)h(to)e(ensure)h (this)f(will)i(not)e(cause)h(an)o(y)e(error.)166 1636 y Fn(Implemen)o(tati)o(on)e(note:)166 1685 y Fq(Assume)e(that)g (broadcast)h(is)f(implemen)o(ted)e(using)i(p)q(oin)o(t-to-p)q(oin)o(t)e (MPI)j(comm)o(unicati)o(on.)j(The)d(follo)o(wing)75 1735 y(t)o(w)o(o)i(rules)i(are)f(satis\014ed:)134 1809 y(1.)22 b(All)13 b(receiv)o(es)i(sp)q(ecify)g(their)f(source)h(explicitly)e (\(no)h(wildcards\).)134 1883 y(2.)22 b(Eac)o(h)12 b(pro)q(cess)i (sends)g(all)d(messages)h(that)h(p)q(ertain)f(to)h(one)f(collectiv)o(e) g(call)g(b)q(efore)h(sending)f(an)o(y)g(message)189 1933 y(that)i(p)q(ertain)g(to)g(a)f(subsequen)o(t)j(collectiv)o(e)e(call.) 166 2007 y(Then)i(messages)g(b)q(elonging)f(to)h(successiv)o(e)i (broadcasts)f(cannot)f(b)q(e)h(confused,)g(as)f(the)h(order)f(of)g(p)q (oin)o(t-)75 2057 y(to-p)q(oin)o(t)d(messages)h(is)g(preserv)o(ed.)20 b(This)14 b(is)f(true,)i(in)e(general,)h(for)f(an)o(y)h(collectiv)o(e)g (library)m(.)166 2196 y FC(A)k(collectiv)o(e)h(comm)o(unication)f(ma)o (y)f(execute)i(in)f(a)g(con)o(text)f(while)i(p)q(oin)o(t-to-p)q(oin)o (t)f(comm)o(uni-)75 2252 y(cations)g(that)g(use)g(the)h(same)f(con)o (text)f(are)h(p)q(ending,)j(or)c(o)q(ccur)i(concurren)o(tly)l(.)30 b(This)19 b(is)g(illustrated)75 2309 y(in)d(example)g(B)g(ab)q(o)o(v)o (e,)f(the)g(\014rst)g(pro)q(cess)h(ma)o(y)f(receiv)o(e)h(a)f(message)g (sen)o(t)g(with)h(the)f(con)o(text)g(of)g(com-)75 2365 y(m)o(unicator)32 b Fv(comm)14 b FC(while)j(it)f(is)g(executing)h(a)f (broadcast)f(with)h(the)g(same)g(comm)o(unicator.)22 b(It)15 b(is)i(the)75 2422 y(implemen)o(tor's)f(resp)q(onsibilit)o(y)h (to)e(ensure)h(this)f(will)i(not)e(cause)g(an)o(y)g(confusion.)166 2554 y Fn(Implemen)o(tati)o(on)21 b(note:)65 b Fq(Assume)22 b(that)f(collectiv)o(e)g(comm)o(unications)d(are)k(implemen)o(ted)d (using)75 2604 y(p)q(oin)o(t-to-p)q(oin)o(t)e(MPI)i(comm)o(unication.) 29 b(Then,)20 b(in)e(order)i(to)e(a)o(v)o(oid)g(confusion,)h(whenev)o (er)h(a)e(comm)o(unica-)75 2654 y(tor)g(is)f(created,)i(a)e(\\hidden)h (comm)o(unicator")c(need)19 b(b)q(e)f(created)h(for)e(collectiv)o(e)g (comm)o(unication.)26 b(A)17 b(direct)75 2704 y(implemen)o(tatio)o(n)11 b(of)i(MPI)h(collectiv)o(e)g(comm)o(unicatio)o(n)d(can)j(ac)o(hiev)o(e) g(a)g(similar)d(e\013ect)k(more)e(c)o(heaply)m(,)g(e.g.,)f(b)o(y)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 103 111 103 110 bop 75 -100 a Fs(4.6.)34 b(CORRECTNESS)1280 b FC(103)75 45 y Fq(using)15 b(a)g(hidden)h(tag)f(or)g(con)o(text)h(bit)g (to)f(indicate)g(whether)i(the)f(comm)o(unicator)d(is)i(used)h(for)f(p) q(oin)o(t-to-p)q(oin)o(t)75 95 y(or)f(collectiv)o(e)g(comm)o(unicatio)o (n.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 104 112 104 111 bop 75 357 a Fu(Chapter)34 b(5)75 565 y Fy(Groups,)40 b(Con)m(texts,)75 690 y(Comm)m(unicators,)g(and)g(Cac)m(hing)75 931 y Ft(5.1)59 b(Intro)r(duction)75 1034 y FC(W)l(e)18 b(b)q(egin)h(this)g(c)o(hapter)f(with)g(a)g(p)q(ersp)q(ectiv)o(e)i(of)d (wh)o(y)h(MPI)g(has)g(included)j(features)c(that)h(are)f(not)75 1091 y(found)11 b(in)h(existing)f(v)o(endor)g(systems,)g(and)f(wh)o(y)h (these)g(are)f(imp)q(ortan)o(t.)18 b(One)12 b(of)e(the)h(main)g(motiv)m (ations)75 1147 y(for)f(the)h(features)g(describ)q(ed)i(here)f(is)f(to) f(allo)o(w)i(for)e(the)h(systematic)g(dev)o(elopmen)o(t)h(of)e (parallel)j(libraries.)75 1273 y Fp(5.1.1)49 b(Why)16 b(w)o(e)g(need)g(lib)o(ra)o(ries)75 1360 y FC(P)o(arallel)k(libraries)g (are)e(needed)i(to)e(encapsulate)i(the)f(distracting)g(complications)h (asso)q(ciated)f(with)75 1417 y(distributed)f(realizations)f(of)e(k)o (ey)h(algorithms.)23 b(They)16 b(ensure)h(consisten)o(t)f(correctness)g (of)g(suc)o(h)g(pro-)75 1473 y(cedures,)h(and)g(pro)o(vide)g(a)f (\\higher)h(lev)o(el")h(of)e(p)q(ortabilit)o(y)h(than)g(MPI)f(itself)i (can)e(pro)o(vide.)25 b(As)16 b(suc)o(h,)75 1529 y(libraries)24 b(prev)o(en)o(t)e(eac)o(h)h(programmer)e(from)h(rep)q(eating)h(the)g(w) o(ork)f(of)g(de\014ning)i(consisten)o(t)f(data)75 1586 y(structures,)13 b(data)f(la)o(y)o(outs,)h(and)g(metho)q(ds)g(that)g (implemen)o(t)h(k)o(ey)f(algorithms)g(\(suc)o(h)g(as)g(matrix)g(op)q (er-)75 1642 y(ations\).)21 b(Since)c(the)f(b)q(est)g(libraries)i(come) d(with)i(sev)o(eral)e(v)m(ariations)i(on)e(parallel)j(systems)d (\(di\013eren)o(t)75 1699 y(data)d(la)o(y)o(outs,)g(di\013eren)o(t)h (strategies)f(dep)q(enden)o(t)i(on)f(size)g(of)g(system,)f(or)g(t)o(yp) q(e)h(of)f(\015oating)h(p)q(oin)o(t\),)g(this)75 1755 y(to)q(o)i(needs)h(to)e(b)q(e)i(hidden)h(from)d(the)i(user.)75 1881 y Fp(5.1.2)49 b(Why)16 b(existing)h(systems)e(do)i(not)f(supp)q(o) o(rt)f(lib)o(ra)o(ries)75 1968 y FC(It)f(is)g(di\016cult)h(to)e(write)h (libraries)h(with)f(message)g(passing)g(systems)f(that)g(v)o(endors)g (pro)q(duce,)i(b)q(ecause)75 2025 y(they)h(ha)o(v)o(e)g(no)g(w)o(a)o(y) f(for)g(the)h(library)h(to)f(insulate)h(itself)g(e\013ectiv)o(ely)g (from)e(the)i(other)e(p)q(oin)o(t-to-p)q(oin)o(t)75 2081 y(message)j(passing)h(that)e(go)q(es)i(on)f(in)h(the)g(system.)28 b(F)l(or)18 b(instance,)i(message)e(tags)f(are)h(a)g(means)h(for)75 2138 y(a)i(library)i(to)e(hide)i(its)e(messages)g(from)g(the)h(user.)39 b(Ho)o(w)o(ev)o(er,)22 b(man)o(y)f(libraries)i(ma)o(y)e(c)o(ho)q(ose)h (the)75 2194 y(same)16 b(tags,)f(or)h(the)g(v)o(endor's)g(library)h(ma) o(y)e(itself)i(indiscriminately)j(use)c(tags)f(when)i(implemen)o(ting) 75 2251 y(collectiv)o(e)f(op)q(erations)e(\()p Fr(e.g.)p FC(,)g(global)h(sum\);)e(in)j(that)d(case,)h(collectiv)o(e)i(and)f(p)q (oin)o(t-to-p)q(oin)o(t)f(messages)75 2307 y(ma)o(y)g(get)h(mixed)h (up.)166 2364 y(Collectiv)o(e)c(op)q(erations,)g(suc)o(h)g(as)f (reduce,)h(broadcast,)f(and)g(so)g(on,)h(are)f(not)f(de\014ned)j(in)f (all)g(v)o(endor)75 2421 y(systems,)j(and)i(when)f(de\014ned)i(are)d (most)h(often)f(only)i(a)o(v)m(ailable)h(o)o(v)o(er)d(the)h(full)i(set) d(of)h(pro)q(cessors)g(\(or)75 2477 y(pro)q(cesses\))d(allo)q(cated)h (to)e(the)i(user.)19 b(Libraries)14 b(need)g(to)e(w)o(ork)g(on)h (di\013eren)o(t)h(collections)g(of)f(pro)q(cesses,)75 2534 y(and)j(to)g(ha)o(v)o(e)g(a)o(v)m(ailable)h(collectiv)o(e)h(op)q (erations)e(that)g(only)h(impact)f(the)g(pro)q(cesses)h(that)e(care)h (ab)q(out)75 2590 y(the)f(collectiv)o(e)i(op)q(eration.)166 2647 y(A)k(third)g(issue)h(is)f(that)f(libraries)j(don't)d(w)o(an)o(t)g (to)g(describ)q(e)j(p)q(oin)o(t-to-p)q(oin)o(t)e(comm)o(unication)75 2704 y(in)f(terms)e(of)g(lo)o(w-lev)o(el,)i(hardw)o(are-dep)q(enden)o (t)g(names.)30 b(In)20 b(fact,)e(they)h(migh)o(t)f(lik)o(e)i(to)e(use)h (virtual)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 105 113 105 112 bop 75 -100 a Fs(5.1.)34 b(INTR)o(ODUCTION)1256 b FC(105)75 45 y(names)13 b(for)f(pro)q(cesses)i(that)e(someho)o(w)g (re\015ect)i(the)f(relationship)i(of)d(pro)q(cesses)i(to)e(one)h (another)g(in)h(the)75 102 y(distributed)20 b(data)e(structure)g (natural)g(to)g(an)g(algorithm.)29 b(Libraries)20 b(\(or)d(program)h (mo)q(dules\))h(that)75 158 y(execute)h(concurren)o(tly)f(on)g (di\013eren)o(t)g(pro)q(cesses)g(ma)o(y)f(w)o(an)o(t)g(to)g(use)h(the)g (same)g(pro)q(cess)g(names)g(for)75 214 y(their)d(in)o(ternal)g(comm)o (unication.)75 337 y Fp(5.1.3)49 b(What)16 b(features)f(a)o(re)g (needed)h(to)g(supp)q(o)o(rt)g(lib)o(ra)o(ries)75 423 y FC(So,)f(w)o(e)g(ha)o(v)o(e)f(established)j(three)f(k)o(ey)f(areas)f (that)h(will)i(supp)q(ort)e(the)g(creation)h(of)e(libraries:)143 517 y FA(\017)23 b FC(Safe)11 b(comm)o(unication)i(space,)f(that)f (guaran)o(tees)g(that)g(a)g(library)i(can)f(comm)o(unicate)g(as)f(it)h (needs)189 574 y(to,)143 668 y FA(\017)23 b FC(Group)14 b(scop)q(e)h(for)g(collectiv)o(e)h(op)q(erations,)f(that)f(allo)o(w)h (libraries)h(to)e(a)o(v)o(oid)h(unnecessarily)i(syn-)189 725 y(c)o(hronizing)f(unin)o(v)o(olv)o(ed)h(pro)q(cessors,)143 819 y FA(\017)23 b FC(Abstract)15 b(pro)q(cess)h(naming)h(to)e(allo)o (w)h(libraries)i(to)d(describ)q(e)j(their)e(comm)o(unication)h(in)g (terms)189 875 y(suitable)f(to)f(their)g(o)o(wn)g(data)g(structures.)75 998 y Fp(5.1.4)49 b(What)16 b(these)f(features)g(a)o(re)g(called)i(in)g (MPI)75 1084 y FC(The)e(features)g(that)g(implemen)o(t)h(these)g(imp)q (ortan)o(t)e(concepts)i(are)f(as)g(follo)o(ws,)g(in)h(MPI:)143 1178 y FA(\017)23 b Fx(Con)o(texts)18 b FC(pro)o(vide)j(the)e(abilit)o (y)i(to)e(ha)o(v)o(e)g(separate)g(safe)h(\\univ)o(erses")g(of)f (message-passing)189 1235 y(in)g(MPI.)g(A)g(con)o(text)f(is)h (something)g(lik)o(e)h(an)f(additional)i(tag)c(that)i(di\013eren)o (tiates)g(messages,)189 1291 y(but)g(the)f(system)h(manages)f(it,)h (and)g(the)g(user)g(nev)o(er)g(manipulates)h(con)o(texts.)30 b(W)l(e)19 b(sa)o(y)f(that)189 1348 y(comm)o(unicators)11 b(\(b)q(elo)o(w\))g(are)g(a)g(sp)q(eci\014c)j(con)o(text)c(of)h(comm)o (unication,)i(when)f(w)o(e)f(are)g(sp)q(eaking)189 1404 y(lo)q(osely)l(.)21 b(The)15 b(use)g(of)g(separate)g(comm)o(unication)g (con)o(texts)g(b)o(y)g(distinct)h(libraries)h(\(or)d(distinct)189 1461 y(library)e(in)o(v)o(o)q(cations\))g(will)h(insulate)g(comm)o (unication)g(in)o(ternal)f(to)g(the)f(library)i(execution)g(from)189 1517 y(external)18 b(comm)o(unication.)29 b(This)18 b(allo)o(ws)h(the)f (in)o(v)o(o)q(cation)g(of)g(the)g(library)g(ev)o(en)h(if)f(there)g(are) 189 1574 y(p)q(ending)c(comm)o(unications,)f(and)f(a)o(v)o(oids)g(the)h (need)g(to)e(sync)o(hronize)j(en)o(try)d(or)h(exit)h(in)o(to)f(library) 189 1630 y(co)q(de.)143 1724 y FA(\017)23 b Fx(Groups)10 b FC(de\014ne)i(an)f(ordered)g(collection)i(of)e(pro)q(cesses,)h(eac)o (h)f(with)g(a)g(rank,)g(and)g(it)g(is)h(this)f(group)189 1781 y(that)k(de\014nes)i(the)f(lo)o(w-lev)o(el)h(names)f(for)g(in)o (ter-pro)q(cess)g(comm)o(unication)h(\(ranks)e(are)h(used)h(for)189 1837 y(sending)j(and)f(receiving\).)34 b(Th)o(us,)20 b(groups)e(de\014ne)j(a)e(scop)q(e)h(for)e(pro)q(cess)i(names)f(in)h(p) q(oin)o(t)g(to)189 1894 y(p)q(oin)o(t)15 b(comm)o(unication.)21 b(In)16 b(addition,)g(groups)f(de\014ne)h(the)f(scop)q(e)h(of)f (collectiv)o(e)i(op)q(erations.)143 1988 y FA(\017)23 b Fx(Comm)o(unicators)15 b FC(\(see)h([13)o(]\))f(encapsulate)i(all)g (of)f(these)g(ideas)h(in)g(an)f(ob)s(ject)f(that)g(pro)o(vides)189 2045 y(the)g(righ)o(t)g(scop)q(e)h(for)e(all)i(comm)o(unication)g(op)q (erations)g(in)g(MPI.)166 2139 y(The)f(curren)o(t)g(practice)h(in)f (man)o(y)g(comm)o(unication)g(libraries)i(is)e(that)g(there)g(is)g(a)g (a)f(unique,)j(pre-)75 2195 y(de\014ned)22 b(comm)o(unication)e(univ)o (erse)h(that)f(includes)i(all)f(pro)q(cesses)g(a)o(v)m(ailable)h(when)e (the)h(library)f(is)75 2252 y(initiated;)25 b(the)c(pro)q(cesses)g(are) f(assigned)i(consecutiv)o(e)f(ranks.)37 b(P)o(articipan)o(ts)20 b(in)i(a)e(p)q(oin)o(t-to-p)q(oin)o(t)75 2308 y(comm)o(unication)h(are) g(iden)o(ti\014ed)h(b)o(y)f(their)g(rank;)i(a)d(collectiv)o(e)j(comm)o (unication)e(\(suc)o(h)g(as)f(broad-)75 2365 y(cast\))c(in)o(v)o(olv)o (es)h(all)g(pro)q(cesses.)25 b(This)17 b(practice)g(can)g(b)q(e)g (follo)o(w)o(ed)g(in)g(MPI)g(b)o(y)f(using)i(the)e(prede\014ned)75 2421 y(comm)o(unicator)e Fv(MPI)p 456 2421 14 2 v 16 w(COMM)p 614 2421 V 17 w(W)o(ORLD)p FC(.)g Fr(Users)h(who)h(ar)n(e)g (satis\014e)n(d)f(with)h(such)h(pr)n(actic)n(e)e(c)n(an)g(plug)i(in)75 2478 y Fv(MPI)p 160 2478 V 16 w(COMM)p 318 2478 V 16 w(W)o(ORLD)e Fr(wher)n(ever)h(a)f(c)n(ommunic)n(ator)h(ar)n(gument)f (is)g(r)n(e)n(quir)n(e)n(d,)g(and)g(c)n(an)g(c)n(onse)n(quently)75 2534 y(disr)n(e)n(gar)n(d)h(the)g(r)n(est)g(of)g(this)g(chapter.)166 2591 y FC(The)f(discussion)h(has)e(dealt)h(so)f(far)f(with)i Fx(in)o(tra-comm)o(unication)p FC(:)22 b(comm)o(unication)15 b(within)g(a)75 2647 y(group.)27 b(MPI)18 b(also)g(supp)q(orts)g Fx(in)o(ter-comm)o(unication)p FC(:)27 b(comm)o(unication)18 b(across)f(groups.)28 b(When)75 2704 y(an)15 b(application)i(is)f (built)h(b)o(y)e(comp)q(osing)h(sev)o(eral)f(parallel)i(mo)q(dules,)f (it)g(is)g(con)o(v)o(enien)o(t)g(to)e(allo)o(w)i(one)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 106 114 106 113 bop 75 -100 a FC(106)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FC(mo)q(dule)h(to)f(comm)o(unicate)g(with)g(the)h(other)e(using)i (lo)q(cal)g(ranks)f(within)h(the)g(second)f(mo)q(dule)h(for)f(ad-)75 102 y(dressing.)21 b(This)15 b(is)h(esp)q(ecially)h(con)o(v)o(enien)o (t)f(in)g(a)f(clien)o(t-serv)o(er)h(computing)f(paradigm,)g(where)g (either)75 158 y(clien)o(t)f(or)e(serv)o(er)g(are)g(parallel.)21 b(The)12 b(supp)q(ort)h(of)f(in)o(ter-comm)o(unication)i(also)e(pro)o (vides)h(a)f(mec)o(hanism)75 214 y(for)g(the)h(extension)g(of)f(MPI)h (to)f(a)g(dynamic)h(mo)q(del)h(where)f(not)f(all)h(pro)q(cesses)g(are)g (preallo)q(cated)g(at)f(ini-)75 271 y(tialization)j(time.)k(In)14 b(suc)o(h)f(a)g(situation,)h(it)f(b)q(ecomes)h(necessary)f(to)g(supp)q (ort)g(comm)o(unication)h(across)75 327 y(\\univ)o(erses.")28 b(In)o(ter-comm)o(unication)19 b(is)f(supp)q(orted)h(b)o(y)e(ob)s (jects)h(called)h Fx(in)o(ter-comm)o(unicators)p FC(.)75 384 y(These)13 b(bind)h(t)o(w)o(o)e(groups)h(together)f(with)h(a)f(con) o(text)h(shared)g(b)o(y)g(b)q(oth)g(groups.)18 b(MPI)13 b(pro)o(vides)h(mec)o(h-)75 440 y(anisms)f(for)e(creating)i(and)f (manipulating)i(in)o(ter-comm)o(unicators.)19 b(They)12 b(are)g(used)h(in)g(p)q(oin)o(t-to-p)q(oin)o(t)75 497 y(comm)o(unication)k(in)g(the)f(same)g(manner)g(as)f(regular)h Fx(in)o(tra-comm)o(unicators)p FC(.)24 b(Users)16 b(who)f(do)h(not)75 553 y(need)g(in)o(ter-comm)o(unication)g(in)g(their)g(applications)h (can)e(safely)h(ignore)f(this)h(extension.)166 613 y(Comm)o(unicators)f (also)h(pro)o(vide)g(a)f(\\cac)o(hing")h(mec)o(hanism)h(that)e(allo)o (ws)h(one)g(to)f(asso)q(ciate)h(new)75 669 y(attributes)g(with)h(comm)o (unicators,)f(on)g(a)g(par)g(with)h(MPI)f(built-in)j(features.)k(This) 17 b(can)f(b)q(e)h(used)g(b)o(y)75 726 y(adv)m(anced)h(users)e(to)g (further)g(adorn)h(comm)o(unicators,)f(and)g(b)o(y)h(MPI)f(to)g (implemen)o(t)i(some)e(comm)o(u-)75 782 y(nicator)h(functions.)25 b(F)l(or)16 b(example,)h(the)g(virtual)h(top)q(ology)e(functions)h (describ)q(ed)i(in)f(Chapter)e(6)g(are)75 839 y(lik)o(ely)h(to)e(b)q(e) g(supp)q(orted)h(this)g(w)o(a)o(y)l(.)75 1000 y Ft(5.2)59 b(Basic)19 b(Concepts)75 1109 y Fp(5.2.1)49 b(Groups)75 1201 y FC(A)17 b Fx(group)g FC(is)g(an)g(ordered)g(set)f(of)h(pro)q (cess)g(iden)o(ti\014ers)h(\(henceforth)f(pro)q(cesses\);)g(pro)q(cess) g(iden)o(ti\014ers)75 1258 y(are)11 b(implemen)o(tation-dep)q(enden)o (t)i(ob)s(jects.)18 b(Eac)o(h)11 b(pro)q(cess)g(in)h(a)f(group)f(is)i (asso)q(ciated)f(with)g(an)g(in)o(teger)75 1314 y Fx(rank)p FC(,)18 b(starting)g(from)f(zero.)29 b(Groups)17 b(are)h(represen)o (ted)h(b)o(y)f(opaque)g Fx(group)j(ob)s(jects)p FC(,)e(and)f(hence)75 1371 y(cannot)k(b)q(e)h(directly)g(transferred)f(from)f(one)h(pro)q (cess)h(to)e(another.)41 b(A)22 b(group)g(is)g(used)h(within)g(a)75 1427 y(comm)o(unicator)13 b(to)f(describ)q(e)j(the)e(participan)o(ts)h (in)g(a)f(comm)o(unication)g(\\univ)o(erse")h(and)f(to)g(rank)g(suc)o (h)75 1484 y(participan)o(ts)j(\(th)o(us)e(giving)i(them)g(unique)g (names)f(within)i(that)d(\\univ)o(erse")i(of)f(comm)o(unication\).)166 1626 y Fn(Implemen)o(tati)o(on)h(note:)48 b Fq(A)17 b(group)g(ma)o(y)e (b)q(e)i(represen)o(ted)j(b)o(y)d(a)f(virtual-to-real)g(pro)q (cess-address-)75 1682 y(translation)d(table.)18 b(Eac)o(h)c(comm)o (unicator)d(ob)r(ject)k(w)o(ould)e(ha)o(v)o(e)h(a)f(p)q(oin)o(ter)i(to) e(suc)o(h)i(a)f(table.)75 1905 y Fp(5.2.2)49 b(Contexts)75 1997 y FC(A)15 b Fx(con)o(text)h FC(is)g(the)f(MPI)g(mec)o(hanism)h (for)f(partitioning)h(comm)o(unication)g(space.)21 b(A)15 b(de\014ning)i(prop-)75 2053 y(ert)o(y)d(of)g(a)g(con)o(text)g(is)h (that)e(a)h(message)g(sen)o(t)h(in)g(a)f(con)o(text)g(cannot)g(b)q(e)h (receiv)o(ed)h(in)f(another)f(con)o(text.)75 2110 y(Con)o(texts)20 b(are)h(not)g(explicit)j(MPI)d(ob)s(jects;)j(they)d(app)q(ear)h(only)g (along)f(with)h(groups)f(in)h(comm)o(u-)75 2166 y(nicators.)31 b(Distinct)20 b(comm)o(unicators)e(in)i(the)f(same)f(pro)q(cess)i(ha)o (v)o(e)e(distinct)i(con)o(texts.)31 b(A)19 b(con)o(text)75 2223 y(is)g(essen)o(tially)h(a)e(tag)g(\(or)g(tags\))f(needed)j(to)e (mak)o(e)g(a)g(comm)o(unicator)g(safe)h(for)e(p)q(oin)o(t-to-p)q(oin)o (t)j(and)75 2279 y(MPI-de\014ned)d(collectiv)o(e)g(comm)o(unication.) 166 2421 y Fn(Implemen)o(tati)o(on)11 b(note:)35 b Fq(A)14 b(p)q(ossible)f(implemen)o(tatio)o(n)d(is)j(for)g(a)g(con)o(text)h(to)e (b)q(e)i(a)f(lab)q(el)f(attac)o(hed)i(to)75 2478 y(messages)f(and)f (matc)o(hed)g(on)g(receiv)o(e.)19 b(Eac)o(h)13 b(in)o(tra-comm)o (unicator)d(stores)j(the)h(v)n(alue)e(of)g(that)g(lab)q(el.)17 b(Comm)o(u-)75 2534 y(nicator)f(generating)g(functions)h(use)f(a)g (collectiv)o(e)g(comm)o(unication)d(to)j(agree)h(on)f(a)g(new)g (group-wide)g(unique)75 2591 y(con)o(text.)25 b(In)15 b(in)o(ter-comm)o(unication,)e(it)i(is)h(more)f(con)o(v)o(enien)o(t)h (to)g(use)g(t)o(w)o(o)g(lab)q(els,)f(one)h(used)h(b)o(y)f(group)f(A)h (to)75 2647 y(send)e(and)e(group)h(B)g(to)g(receiv)o(e,)h(and)e(the)i (second)f(used)h(b)o(y)f(group)f(B)h(to)g(send)h(and)e(group)h(A)g(to)f (receiv)o(e.)19 b(Since)75 2704 y(con)o(texts)c(are)f(not)g(explicit)g (ob)r(jects,)g(other)h(implem)o(en)o(tations)c(are)k(p)q(ossible.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 107 115 107 114 bop 75 -100 a Fs(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b FC(107)75 45 y Fp(5.2.3)49 b(Communicato)o(rs)75 131 y FC(Comm)o(unicators)18 b(bring)h(together)f(the)h(concepts)h(of)e (group)g(and)h(con)o(text.)30 b(F)l(urthermore,)19 b(to)f(sup-)75 188 y(p)q(ort)c(implemen)o(tation-sp)q(eci\014c)j(optimizations,)d(and) g(virtual)h(top)q(ologies,)f(they)g(\\cac)o(he")g(additional)75 244 y(information.)37 b(The)21 b(source)g(and)g(destination)h(of)e(a)g (message)h(is)g(iden)o(ti\014ed)i(b)o(y)e(the)g(rank)f(of)h(that)75 301 y(pro)q(cess)14 b(within)g(the)g(group.)19 b(F)l(or)13 b(collectiv)o(e)i(comm)o(unication,)f(the)f(comm)o(unicator)g(sp)q (eci\014es)j(the)d(set)75 357 y(of)k(pro)q(cesses)h(that)f(participate) i(in)f(the)g(collectiv)o(e)i(op)q(eration)d(and)h(their)g(order,)g (when)g(signi\014can)o(t.)75 414 y(Th)o(us,)c(the)g(comm)o(unicator)g (restricts)f(the)h(\\spatial")g(scop)q(e)h(of)f(comm)o(unication,)g (and)g(pro)o(vides)h(lo)q(cal)75 470 y(pro)q(cess)g(addressing.)166 527 y(Comm)o(unicators)10 b(are)h(represen)o(ted)h(b)o(y)g(opaque)f Fx(comm)o(unicator)i(ob)s(jects)p FC(,)f(and)g(hence)g(cannot)75 583 y(b)q(e)k(directly)g(transferred)f(from)g(one)g(pro)q(cess)g(to)g (another.)75 706 y Fp(5.2.4)49 b(Prede\014ned)15 b(Communicato)o(rs)75 793 y FC(An)g(initial)h(comm)o(unicator)27 b Fl(MPI)p 665 793 13 2 v 14 w(COMM)p 809 793 V 15 w(W)o(ORLD)14 b FC(of)g(all)h(pro)q(cesses)g(the)g(lo)q(cal)g(pro)q(cess)g(can)f (comm)o(u-)75 849 y(nicate)i(with)f(after)g(initialization)j(\(itself)e (included\))h(is)f(de\014ned)h(once)e Fv(MPI)p 1403 849 14 2 v 16 w(INIT)f FC(has)h(b)q(een)i(called.)166 906 y(In)f(a)f(static-pro)q(cess-mo)q(del)h(implemen)o(tation)g(of)f(MPI,)g (all)h(pro)q(cesses)g(that)e(participate)i(in)g(the)75 962 y(computation)i(are)g(a)o(v)m(ailable)i(after)d(MPI)i(is)f (initialize)q(d.)32 b(F)l(or)17 b(this)i(case,)g Fl(MPI)p 1481 962 13 2 v 14 w(COMM)p 1625 962 V 14 w(W)o(ORLD)g FC(is)f(a)75 1019 y(comm)o(unicator)h(of)g(all)i(pro)q(cesses)f(a)o(v)m (ailable)h(for)e(the)g(computation;)i(this)f(comm)o(unicator)g(has)f (the)75 1075 y(same)14 b(v)m(alue)h(in)g(all)g(pro)q(cesses.)20 b(In)15 b(an)f(implemen)o(tation)h(of)f(MPI)g(where)g(pro)q(cesses)h (can)f(dynamically)75 1132 y(join)23 b(an)f(MPI)g(execution,)j(it)d(ma) o(y)g(b)q(e)h(the)f(case)h(that)e(a)h(pro)q(cess)h(starts)e(an)h(MPI)g (computation)75 1188 y(without)d(ha)o(ving)g(access)g(to)f(all)i(other) f(pro)q(cesses.)31 b(In)20 b(suc)o(h)f(situations,)h Fl(MPI)p 1479 1188 V 14 w(COMM)p 1623 1188 V 15 w(W)o(ORLD)f FC(is)g(a)75 1245 y(comm)o(unicator)d(incorp)q(orating)h(all)h(pro)q (cesses)f(with)g(whic)o(h)g(the)g(joining)h(pro)q(cess)f(can)f (immediately)75 1301 y(comm)o(unicate.)33 b(Therefore,)20 b Fl(MPI)p 683 1301 V 14 w(COMM)p 827 1301 V 15 w(W)o(ORLD)f FC(ma)o(y)g(sim)o(ultaneouly)i(ha)o(v)o(e)e(di\013eren)o(t)h(v)m(alues) g(in)75 1358 y(di\013eren)o(t)15 b(pro)q(cesses.)166 1414 y(All)i(MPI)e(implemen)o(tations)i(are)e(required)i(to)d(pro)o (vide)i(the)g Fl(MPI)p 1316 1414 V 14 w(COMM)p 1460 1414 V 15 w(W)o(ORLD)f FC(comm)o(unica-)75 1471 y(tor.)34 b(It)21 b(cannot)f(b)q(e)h(deallo)q(cated)h(during)f(the)f(life)i(of)e (a)g(pro)q(cess.)36 b(The)20 b(group)g(corresp)q(onding)i(to)75 1527 y(this)14 b(comm)o(unicator)g(do)q(es)g(not)g(app)q(ear)g(as)g(a)f (pre-de\014ned)j(constan)o(t,)d(but)h(it)h(ma)o(y)e(b)q(e)i(accessed)f (using)75 1584 y Fv(MPI)p 160 1584 14 2 v 16 w(COMM)p 318 1584 V 16 w(GROUP)e FC(\(see)f(b)q(elo)o(w\).)18 b(MPI)11 b(do)q(es)f(not)h(sp)q(ecify)g(the)g(corresp)q(ondence)h(b)q (et)o(w)o(een)e(the)h(pro-)75 1640 y(cess)g(rank)g(in)g Fl(MPI)p 388 1640 13 2 v 14 w(COMM)p 532 1640 V 15 w(W)o(ORLD)g FC(and)g(its)g(\(mac)o(hine-dep)q(enden)o(t\))h(absolute)g(address.)18 b(Neither)12 b(do)q(es)75 1697 y(MPI)k(sp)q(ecify)i(the)e(function)h (of)e(the)i(host)e(pro)q(cess,)h(if)h(an)o(y)l(.)23 b(An)16 b(inquiry)i(function)f Fv(MPI)p 1636 1697 14 2 v 15 w(GET)p 1739 1697 V 17 w(HOST)75 1753 y FC(is)i(pro)o(vided)g(in)g(Chapter)f(7) g(so)g(that)f(MPI)i(programs)e(can)h(determine)h(if)g(there)g(is)f(a)g (host)g(pro)q(cess,)75 1809 y(and)13 b(\014nd)g(its)g(rank)f(in)h Fl(MPI)p 539 1809 13 2 v 15 w(COMM)p 684 1809 V 14 w(W)o(ORLD)g FC(if)g(it)g(should)g(exist.)20 b(Other)12 b(implemen)o(tation-dep)q (end)q(en)o(t,)75 1866 y(prede\014ned)17 b(comm)o(unicators)e(ma)o(y)f (also)h(b)q(e)h(pro)o(vided.)75 2011 y Ft(5.3)59 b(Group)20 b(Management)75 2113 y FC(This)g(section)h(describ)q(es)g(the)f (manipulation)h(of)e(pro)q(cess)h(groups)g(in)g(MPI.)f(These)h(op)q (erations)g(are)75 2169 y(lo)q(cal)c(and)g(their)f(execution)i(do)e (not)g(require)h(in)o(terpro)q(cess)f(comm)o(unication.)75 2292 y Fp(5.3.1)49 b(Group)16 b(Accesso)o(rs)75 2426 y Fv(MPI)p 160 2426 14 2 v 16 w(GROUP)p 328 2426 V 18 w(SIZE\(group,)e(size\))117 2503 y Fq(IN)155 b Fv(group)479 b Fq(group)14 b(\(handle\))117 2579 y(OUT)108 b Fv(size)519 b Fq(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)g(\(in)o (teger\))75 2704 y Fo(int)23 b(MPI)p 245 2704 15 2 v 17 w(Group)p 382 2704 V 17 w(size\(MPI)p 591 2704 V 16 w(Group)g(group,)g(int)g(*size\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 108 116 108 115 bop 75 -100 a FC(108)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(SIZE\(GROUP,)23 b(SIZE,)g(IERROR\))170 102 y(INTEGER)g(GROUP,)g (SIZE,)h(IERROR)75 236 y Fv(MPI)p 160 236 14 2 v 16 w(GROUP)p 328 236 V 18 w(RANK\(group,)14 b(rank\))117 313 y Fq(IN)155 b Fv(group)479 b Fq(group)14 b(\(handle\))117 389 y(OUT)108 b Fv(rank)504 b Fq(rank)50 b(of)g(the)h(calling)e(pro)q(cess)j(in)e (group,)59 b(or)905 446 y Fl(MPI)p 982 446 13 2 v 15 w(UNDEFINED)13 b Fq(if)h(the)h(pro)q(cess)i(is)d(not)h(a)f(mem)o(b)q (er)f(\(in-)905 502 y(teger\))75 627 y Fo(int)23 b(MPI)p 245 627 15 2 v 17 w(Group)p 382 627 V 17 w(rank\(MPI)p 591 627 V 16 w(Group)g(group,)g(int)g(*rank\))75 714 y(MPI)p 150 714 V 17 w(GROUP)p 287 714 V 16 w(RANK\(GROUP,)g(RANK,)g (IERROR\))170 770 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)75 904 y Fv(MPI)p 160 904 14 2 v 16 w(GROUP)p 328 904 V 18 w(TRANSLA)l(TE)p 602 904 V 17 w(RANKS)16 b(\(group1,)e(n,)h(ranks1,) g(group2,)g(ranks2\))117 982 y Fq(IN)155 b Fv(group1)456 b Fq(group1)14 b(\(handle\))117 1058 y(IN)155 b Fv(n)564 b Fq(n)o(um)o(b)q(er)11 b(of)h(ranks)g(in)23 b Fl(ranks1)13 b Fq(and)f Fl(ranks2)g Fq(arra)o(ys)g(\(in)o(teger\))117 1134 y(IN)155 b Fv(ranks1)464 b Fq(arra)o(y)14 b(of)f(zero)i(or)f(more) f(v)n(alid)f(ranks)i(in)g(group1)117 1209 y(IN)155 b Fv(group2)456 b Fq(group2)14 b(\(handle\))117 1285 y(OUT)108 b Fv(ranks2)464 b Fq(arra)o(y)64 b(of)f(corresp)q(onding)i(ranks)f(in)g (group2,)905 1342 y Fl(MPI)p 982 1342 13 2 v 15 w(UNDEFINED)12 b Fq(when)j(no)e(corresp)q(ondence)k(exists.)75 1467 y Fo(int)23 b(MPI)p 245 1467 15 2 v 17 w(Group)p 382 1467 V 17 w(translate)p 615 1467 V 16 w(ranks)g(\(MPI)p 870 1467 V 16 w(Group)h(group1,)f(int)g(n,)h(int)f(*ranks1,)393 1523 y(MPI)p 468 1523 V 17 w(Group)g(group2,)g(int)g(*ranks2\))75 1610 y(MPI)p 150 1610 V 17 w(GROUP)p 287 1610 V 16 w(TRANSLATE)p 519 1610 V 16 w(RANKS\(GROUP1,)f(N,)i(RANKS1,)f(GROUP2,)g(RANKS2,)g (IERROR\))170 1666 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1\(*\),)e(GROUP2,)h (RANKS2\(*\),)g(IERROR)75 1820 y Fp(5.3.2)49 b(Group)16 b(Constructo)o(rs)75 1907 y FC(These)k(functions)g(allo)o(w)g (construction)g(of)g(new)g(groups)f(from)g(existing)h(groups.)33 b(These)20 b(are)g(lo)q(cal)75 1963 y(op)q(erations,)d(and)h(distinct)g (groups)f(ma)o(y)f(b)q(e)i(de\014ned)h(on)e(di\013eren)o(t)g(pro)q (cesses;)h(a)f(pro)q(cess)h(ma)o(y)e(also)75 2020 y(de\014ne)f(a)f (group)g(that)f(do)q(es)i(not)f(include)i(itself.)21 b(Consisten)o(t)14 b(de\014nitions)i(are)d(required)j(when)e(groups)75 2076 y(are)h(used)g(as)g(argumen)o(ts)f(in)i(comm)o(unicator-building)i (functions.)i(MPI)15 b(do)q(es)h(not)e(pro)o(vide)i(a)f(mec)o(h-)75 2132 y(anism)h(to)g(build)i(a)e(group)g(from)f(scratc)o(h,)h(but)g (only)g(from)g(other,)f(previously)j(de\014ned)f(groups.)23 b(The)75 2189 y(base)13 b(group,)g(up)q(on)h(whic)o(h)g(all)g(other)f (groups)g(are)g(de\014ned,)i(is)e(the)h(group)f(asso)q(ciated)g(with)h (the)f(initial)75 2245 y(comm)o(unicator)e Fv(MPI)p 453 2245 14 2 v 16 w(COMM)p 611 2245 V 17 w(W)o(ORLD)g FC(\(accessible)i (through)f(the)g(function)g Fv(MPI)p 1520 2245 V 16 w(COMM)p 1678 2245 V 16 w(GROUP)p FC(\).)75 2349 y Fv(MPI)p 160 2349 V 16 w(GROUP)p 328 2349 V 18 w(UNION\(group1,)i(group2,)h (newgroup\))117 2427 y Fq(IN)155 b Fv(group1)456 b Fq(\014rst)15 b(group)f(\(handle\))117 2503 y(IN)155 b Fv(group2)456 b Fq(second)15 b(group)f(\(handle\))117 2579 y(OUT)108 b Fv(newgroup)405 b Fq(union)14 b(group)f(\(handle\))75 2704 y Fo(int)23 b(MPI)p 245 2704 15 2 v 17 w(Group)p 382 2704 V 17 w(union\(MPI)p 615 2704 V 16 w(Group)g(group1,)g(MPI)p 1037 2704 V 17 w(Group)g(group2,)g(MPI)p 1460 2704 V 16 w(Group)h(*newgroup\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 109 117 109 116 bop 75 -100 a Fs(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b FC(109)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(UNION\(GROUP1,)22 b(GROUP2,)h(NEWGROUP,)g(IERROR\))170 102 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 250 y Fv(MPI)p 160 250 14 2 v 16 w(GROUP)p 328 250 V 18 w(INTERSECTION\(group1,)15 b(group2,)f(newgroup\))117 342 y Fq(IN)155 b Fv(group1)456 b Fq(\014rst)15 b(group)f(\(handle\)) 117 447 y(IN)155 b Fv(group2)456 b Fq(second)15 b(group)f(\(handle\)) 117 551 y(OUT)108 b Fv(newgroup)405 b Fq(in)o(tersection)15 b(group)f(\(handle\))75 691 y Fo(int)23 b(MPI)p 245 691 15 2 v 17 w(Group)p 382 691 V 17 w(intersection\(MPI)p 783 691 V 15 w(Group)g(group1,)g(MPI)p 1204 691 V 17 w(Group)g(group2,)393 747 y(MPI)p 468 747 V 17 w(Group)g(*newgroup\))75 848 y(MPI)p 150 848 V 17 w(GROUP)p 287 848 V 16 w (INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 905 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1053 y Fv(MPI)p 160 1053 14 2 v 16 w(GROUP)p 328 1053 V 18 w(DIFFERENCE\(group1,)14 b(group2,)g(newgroup\))117 1145 y Fq(IN)155 b Fv(group1)456 b Fq(\014rst)15 b(group)f(\(handle\)) 117 1250 y(IN)155 b Fv(group2)456 b Fq(second)15 b(group)f(\(handle\)) 117 1355 y(OUT)108 b Fv(newgroup)405 b Fq(di\013erence)16 b(group)e(\(handle\))75 1494 y Fo(int)23 b(MPI)p 245 1494 15 2 v 17 w(Group)p 382 1494 V 17 w(difference\(MPI)p 735 1494 V 15 w(Group)g(group1,)g(MPI)p 1156 1494 V 17 w(Group)g(group2,)393 1550 y(MPI)p 468 1550 V 17 w(Group)g(*newgroup\)) 75 1652 y(MPI)p 150 1652 V 17 w(GROUP)p 287 1652 V 16 w(DIFFERENCE\(GROUP1,)f(GROUP2,)h(NEWGROUP,)f(IERROR\))170 1708 y(INTEGER)h(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1809 y FC(The)15 b(set-lik)o(e)i(op)q(erations)e(are)g(de\014ned)i(as)d (follo)o(ws:)75 1962 y Fx(union)24 b FC(All)19 b(elemen)o(ts)g(of)f (the)h(\014rst)f(group)g(\()p Fv(group1)p FC(\),)g(follo)o(w)o(ed)h(b)o (y)f(all)h(elemen)o(ts)g(of)f(second)h(group)189 2019 y(\()p Fv(group2)p FC(\))14 b(not)h(in)h(\014rst.)75 2172 y Fx(in)o(tersect)23 b FC(all)18 b(elemen)o(ts)h(of)e(the)g (\014rst)h(group)f(that)g(are)g(also)g(in)i(the)f(second)g(group,)f (ordered)h(as)f(in)189 2228 y(\014rst)d(group.)75 2381 y Fx(di\013erence)22 b FC(all)c(elemen)o(ts)f(of)g(the)g(\014rst)f (group)g(that)g(are)h(not)f(in)h(the)g(second)g(group,)g(ordered)g(as)f (in)189 2438 y(the)f(\014rst)g(group.)75 2591 y(Note)k(that)g(for)g (these)h(op)q(erations)g(the)f(order)h(of)f(pro)q(cesses)h(in)g(the)g (output)f(group)h(is)g(determined)75 2647 y(primarily)g(b)o(y)f(order)g (in)h(the)f(\014rst)g(group)f(\(if)h(p)q(ossible\))i(and)e(then,)h(if)f (necessary)l(,)h(b)o(y)f(order)g(in)h(the)75 2704 y(second)c(group.)j (Neither)d(union)h(nor)e(in)o(tersection)h(are)e(comm)o(utativ)o(e,)h (but)g(b)q(oth)g(are)g(asso)q(ciativ)o(e.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 110 118 110 117 bop 75 -100 a FC(110)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(INCL\(group,)e(n,)j(ranks,)f(newgroup\))117 144 y Fq(IN)155 b Fv(group)479 b Fq(group)14 b(\(handle\))117 262 y(IN)155 b Fv(n)564 b Fq(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(arra)o(y)h (ranks)g(\(and)g(size)h(of)e Fl(new-)905 318 y(group)p Fq(\))f(\(in)o(teger\))117 437 y(IN)155 b Fv(ranks)487 b Fq(ranks)13 b(of)f(pro)q(cesses)i(in)e Fl(group)h Fq(to)f(app)q(ear)h (in)f Fl(newgroup)i Fq(\(ar-)905 493 y(ra)o(y)g(of)f(in)o(tegers\))117 611 y(OUT)108 b Fv(newgroup)405 b Fq(new)12 b(group)f(deriv)o(ed)h (from)e(ab)q(o)o(v)o(e,)h(in)g(the)h(order)h(de\014ned)f(b)o(y)905 668 y Fl(ranks)j Fq(\(handle\))75 814 y Fo(int)23 b(MPI)p 245 814 15 2 v 17 w(Group)p 382 814 V 17 w(incl\(MPI)p 591 814 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 814 V 17 w(Group)h(*newgroup\))75 922 y(MPI)p 150 922 V 17 w(GROUP)p 287 922 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 978 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 1086 y FC(The)17 b(function)h Fv(MPI)p 526 1086 14 2 v 16 w(GROUP)p 694 1086 V 18 w(INCL)f FC(creates)g(a)g(group)f Fv(newgroup)i FC(that)f(consists)g(of)g(the)g Fv(n)h FC(pro-)75 1142 y(cesses)e(in)h Fv(group)f FC(with)h(ranks)e Fv(rank[0],)p Fw(:)8 b(:)g(:)n Fv(,)16 b(rank[n-1])p FC(;)g(the)h(pro)q(cess)f(with)g(rank)g Fv(i)g FC(in)h Fv(newgroup)f FC(is)h(the)75 1199 y(pro)q(cess)e(with)f(rank)h Fv(ranks[i])g FC(in)g Fv(group)p FC(.)k(Eac)o(h)c(of)f(the)g Fv(n)h FC(elemen)o(ts)g(of)f Fv(ranks)h FC(m)o(ust)f(b)q(e)h(a)f(v)m (alid)i(rank)e(in)75 1255 y Fv(group)j FC(and)f(all)i(elemen)o(ts)f(m)o (ust)g(b)q(e)g(distinct,)h(or)e(else)h(the)g(program)f(is)h(erroneous.) 24 b(If)17 b Fv(n)g FC(=)32 b(0,)17 b(then)75 1312 y Fv(newgroup)i FC(is)f(empt)o(y)l(.)29 b(This)18 b(function)h(can,)g (for)e(instance,)i(b)q(e)g(used)g(to)e(reorder)h(the)g(elemen)o(ts)h (of)e(a)75 1368 y(group.)75 1493 y Fv(MPI)p 160 1493 V 16 w(GROUP)p 328 1493 V 18 w(EX)o(CL\(group,)d(n,)h(ranks,)g (newgroup\))117 1592 y Fq(IN)155 b Fv(group)479 b Fq(group)14 b(\(handle\))117 1710 y(IN)155 b Fv(n)564 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 1828 y(IN)155 b Fv(ranks)487 b Fq(arra)o(y)15 b(of)f(in)o(teger)i (ranks)f(in)g Fl(group)g Fq(not)g(to)g(app)q(ear)g(in)g Fl(new-)905 1885 y(group)117 2003 y Fq(OUT)108 b Fv(newgroup)405 b Fq(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 2059 y(de\014ned)d(b)o(y)28 b Fl(group)14 b Fq(\(handle\))75 2205 y Fo(int)23 b(MPI)p 245 2205 15 2 v 17 w(Group)p 382 2205 V 17 w(excl\(MPI)p 591 2205 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 2205 V 17 w(Group)h(*newgroup\))75 2313 y(MPI)p 150 2313 V 17 w(GROUP)p 287 2313 V 16 w(EXCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 2370 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 2478 y FC(The)12 b(function)h Fv(MPI)p 516 2478 14 2 v 16 w(GROUP)p 684 2478 V 17 w(EX)o(CL)f FC(creates)g(a)f(group)h(of)g(pro)q(cesses)g Fv(newgroup)h FC(that)e(is)h(obtained)75 2534 y(b)o(y)k(deleting)i(from)e Fv(group)g FC(those)g(pro)q(cesses)h(with)f(ranks)g Fv(ranks[0])h(,)p Fw(:)8 b(:)g(:)14 b Fv(ranks[n-1])p FC(.)24 b(The)17 b(ordering)f(of)75 2591 y(pro)q(cesses)f(in)h Fv(newgroup)g FC(is)f(iden)o(tical)i(to)e(the)g(ordering)g(in)h Fv(group)p FC(.)j(Eac)o(h)c(of)g(the)g Fv(n)g FC(elemen)o(ts)h(of)e Fv(ranks)75 2647 y FC(m)o(ust)h(b)q(e)h(a)f(v)m(alid)i(rank)e(in)h Fv(group)f FC(and)h(all)g(elemen)o(ts)g(m)o(ust)f(b)q(e)h(distinct;)g (otherwise,)f(the)g(program)f(is)75 2704 y(erroneous.)20 b(If)15 b Fv(n)h FC(=)28 b(0,)14 b(then)i Fv(newgroup)g FC(is)g(iden)o(tical)h(to)d Fv(group)p FC(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 111 119 111 118 bop 75 -100 a Fs(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b FC(111)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(RANGE)p 494 45 V 17 w(INCL\(group,)14 b(n,)h(ranges,)g (newgroup\))117 122 y Fq(IN)155 b Fv(group)479 b Fq(group)14 b(\(handle\))117 197 y(IN)155 b Fv(n)564 b Fq(n)o(um)o(b)q(er)13 b(of)h(triplets)g(in)f(arra)o(y)28 b Fl(ranges)14 b Fq(\(in)o(teger\)) 117 272 y(IN)155 b Fv(ranges)466 b Fq(a)11 b(one-dimensional)e(arra)o (y)i(of)g(in)o(teger)g(triplets,)h(of)f(the)h(form)905 328 y(\(\014rst)i(rank,)e(last)g(rank,)h(stride\))g(indicating)e(ranks) i(in)f Fl(group)905 385 y Fq(or)i(pro)q(cesses)j(to)c(b)q(e)i(included) f(in)f Fl(newgroup)117 460 y Fq(OUT)108 b Fv(newgroup)405 b Fq(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v)o(e,)h(in)g(the)h (order)h(de\014ned)f(b)o(y)905 516 y Fl(ranges)j Fq(\(handle\))75 641 y Fo(int)23 b(MPI)p 245 641 15 2 v 17 w(Group)p 382 641 V 17 w(range)p 519 641 V 16 w(incl\(MPI)p 727 641 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 697 y(MPI)p 468 697 V 17 w(Group)g(*newgroup\))75 784 y(MPI)p 150 784 V 17 w(GROUP)p 287 784 V 16 w(RANGE)p 423 784 V 17 w(INCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 840 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 926 y FC(If)31 b Fv(ranges)15 b FC(consist)g(of)g(the)h(triplets)189 1019 y(\()p Fw(f)5 b(ir)q(st)309 1026 y Fi(1)328 1019 y Fw(;)j(l)q(ast)425 1026 y Fi(1)444 1019 y Fw(;)g(str)q(ide)585 1026 y Fi(1)604 1019 y FC(\))p Fw(;)g(:::;)g FC(\()p Fw(f)d(irst)822 1026 y Fh(n)844 1019 y Fw(;)j(l)q(ast)941 1026 y Fh(n)964 1019 y Fw(;)g(str)q(ide)1105 1026 y Fh(n)1127 1019 y FC(\))75 1111 y(then)16 b Fv(newgroup)f FC(consists)h(of)f(the)g (sequence)h(of)f(pro)q(cesses)h(in)g Fv(group)f FC(with)g(ranks)189 1222 y Fw(f)5 b(ir)q(st)291 1229 y Fi(1)311 1222 y Fw(;)j(f)d(ir)q(st) 434 1229 y Fi(1)463 1222 y FC(+)10 b Fw(str)q(ide)628 1229 y Fi(1)648 1222 y Fw(;)e(:::;)g(f)d(ir)q(st)831 1229 y Fi(1)859 1222 y FC(+)904 1162 y Fg(\026)933 1191 y Fw(l)q(ast)1009 1198 y Fi(1)1039 1191 y FA(\000)11 b Fw(f)5 b(ir)q(st)1187 1198 y Fi(1)p 933 1211 274 2 v 1000 1253 a Fw(str)q(ide)1120 1260 y Fi(1)1212 1162 y Fg(\027)1243 1222 y Fw(str)q(ide)1363 1229 y Fi(1)1383 1222 y Fw(;)j(:::)189 1375 y(f)d(ir)q(st)291 1382 y Fh(n)314 1375 y Fw(;)j(f)d(ir)q(st)437 1382 y Fh(n)471 1375 y FC(+)10 b Fw(str)q(ide)636 1382 y Fh(n)659 1375 y Fw(;)e(:::;)g(f)d(ir) q(st)842 1382 y Fh(n)874 1375 y FC(+)919 1315 y Fg(\026)948 1344 y Fw(l)q(ast)1024 1351 y Fh(n)1058 1344 y FA(\000)10 b Fw(f)5 b(ir)q(st)1205 1351 y Fh(n)p 948 1364 282 2 v 1017 1406 a Fw(str)q(ide)1137 1413 y Fh(n)1234 1315 y Fg(\027)1266 1375 y Fw(str)q(ide)1386 1382 y Fh(n)1409 1375 y Fw(:)75 1496 y FC(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q (e)i(a)f(v)m(alid)h(rank)f(in)h Fv(group)f FC(and)g(all)h(computed)f (ranks)g(m)o(ust)f(b)q(e)i(distinct,)75 1553 y(else)18 b(the)e(program)g(is)h(erroneous.)24 b(Note)16 b(that)g(w)o(e)h(ma)o(y) f(ha)o(v)o(e)g Fw(f)5 b(ir)q(st)1288 1560 y Fh(i)1317 1553 y Fw(>)16 b(l)q(ast)1444 1560 y Fh(i)1458 1553 y FC(,)h(and)g Fw(str)q(ide)1698 1560 y Fh(i)1728 1553 y FC(ma)o(y)f(b)q(e)75 1609 y(negativ)o(e,)f(but)g(cannot)g(b)q(e)h (zero.)166 1666 y(The)i(functionalit)o(y)h(of)e(this)i(routine)f(is)g (sp)q(eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i(the) f(arra)o(y)75 1722 y(of)g(ranges)g(to)g(an)h(arra)o(y)e(of)h(the)h (included)i(ranks)d(and)h(passing)g(the)g(resulting)g(arra)o(y)e(of)i (ranks)f(and)75 1779 y(other)e(argumen)o(ts)g(to)g Fv(MPI)p 560 1779 14 2 v 16 w(GROUP)p 728 1779 V 18 w(INCL)p FC(.)g(A)g(call)i (to)e Fv(MPI)p 1148 1779 V 16 w(GROUP)p 1316 1779 V 18 w(INCL)g FC(is)h(equiv)m(alen)o(t)h(to)e(a)h(call)75 1835 y(to)e Fv(MPI)p 216 1835 V 16 w(GROUP)p 384 1835 V 18 w(RANGE)p 550 1835 V 18 w(INCL)g FC(with)i(eac)o(h)f(rank)g Fv(i)g FC(in)h Fv(ranks)f FC(replaced)i(b)o(y)e(the)g(triplet)h Fo(\(i,i,1\))e FC(in)75 1891 y(the)g(argumen)o(t)g Fv(ranges)p FC(.)75 1995 y Fv(MPI)p 160 1995 V 16 w(GROUP)p 328 1995 V 18 w(RANGE)p 494 1995 V 17 w(EX)o(CL\(group,)g(n,)g(ranges,)g (newgroup\))117 2072 y Fq(IN)155 b Fv(group)479 b Fq(group)14 b(\(handle\))117 2147 y(IN)155 b Fv(n)564 b Fq(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 2222 y(IN)155 b Fv(ranges)466 b Fq(a)13 b(one-dimensional)d(arra)o(y)j (of)f(in)o(teger)h(triplets)g(of)f(the)i(form)905 2278 y(\(\014rst)j(rank,)f(last)g(rank,)g(stride\),)h(indicating)e(the)i (ranks)f(in)905 2335 y Fl(group)22 b Fq(of)f(pro)q(cesses)j(to)d(b)q(e) h(excluded)g(from)e(the)i(output)905 2391 y(group)14 b Fl(newgroup)p Fq(.)117 2466 y(OUT)108 b Fv(newgroup)405 b Fq(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 2523 y(in)c Fl(group)g Fq(\(handle\))75 2647 y Fo(int)23 b(MPI)p 245 2647 15 2 v 17 w(Group)p 382 2647 V 17 w(range)p 519 2647 V 16 w(excl\(MPI)p 727 2647 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 2704 y(MPI)p 468 2704 V 17 w(Group)g(*newgroup\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 112 120 112 119 bop 75 -100 a FC(112)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(RANGE)p 423 45 V 17 w(EXCL\(GROUP,)22 b(N,)i(RANGES,)f(NEWGROUP,)f (IERROR\))170 102 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,) h(IERROR)75 191 y FC(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q(e)i (a)f(v)m(alid)h(rank)f(in)h Fv(group)f FC(and)g(all)h(computed)f(ranks) g(m)o(ust)f(b)q(e)i(distinct,)75 248 y(or)i(else)h(the)f(program)f(is)i (erroneous.)166 308 y(The)i(functionalit)o(y)h(of)e(this)i(routine)f (is)g(sp)q(eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i (the)f(arra)o(y)75 364 y(of)g(ranges)f(to)h(an)g(arra)o(y)f(of)h(the)g (excluded)i(ranks)e(and)g(passing)h(the)f(resulting)h(arra)o(y)e(of)h (ranks)g(and)75 421 y(other)c(argumen)o(ts)g(to)g Fv(MPI)p 554 421 14 2 v 16 w(GROUP)p 722 421 V 18 w(EX)o(CL)p FC(.)g(A)h(call)h(to)e Fv(MPI)p 1146 421 V 16 w(GROUP)p 1314 421 V 18 w(EX)o(CL)g FC(is)i(equiv)m(alen)o(t)g(to)e(a)h(call)75 477 y(to)g Fv(MPI)p 216 477 V 15 w(GROUP)p 383 477 V 18 w(RANGE)p 549 477 V 18 w(EX)o(CL)g FC(with)h(eac)o(h)f(rank)g Fv(i)g FC(in)h Fv(ranks)g FC(replaced)g(b)o(y)f(the)h(triplet)g Fo(\(i,i,1\))e FC(in)75 534 y(the)h(argumen)o(t)g Fv(ranges)p FC(.)166 593 y(Note:)20 b(the)c(range)g(op)q(erations)f(do)h(not)f (explicitly)k(en)o(umerate)d(ranks,)f(and)h(therefore)f(are)g(more)75 650 y(scalable)k(if)f(implemen)o(ted)h(e\016cien)o(tly)l(.)28 b(Hence,)18 b(w)o(e)f(recommend)h(MPI)f(programmers)f(to)h(use)h(them) 75 706 y(whenenev)o(er)e(p)q(ossible,)h(as)d(high-qualit)o(y)j (implemen)o(tations)g(will)g(tak)o(e)d(adv)m(an)o(tage)h(of)f(this)i (fact.)75 848 y Fp(5.3.3)49 b(Group)16 b(Destructo)o(rs)75 987 y Fv(MPI)p 160 987 V 16 w(GROUP)p 328 987 V 18 w(FREE\(group\))117 1068 y Fq(INOUT)62 b Fv(group)479 b Fq(group)14 b(\(handle\))75 1196 y Fo(int)23 b(MPI)p 245 1196 15 2 v 17 w(Group)p 382 1196 V 17 w(free\(MPI)p 591 1196 V 16 w(Group)g(*group\))75 1286 y(MPI)p 150 1286 V 17 w(GROUP)p 287 1286 V 16 w(FREE\(GROUP,)g (IERROR\))170 1342 y(INTEGER)g(GROUP,)g(IERROR)166 1432 y FC(This)12 b(op)q(eration)g(marks)f(a)g(group)g(ob)s(ject)g(for)g (deallo)q(cation.)20 b(The)12 b(handle)h Fv(group)f FC(is)g(set)f(to)g (n)o(ull)i(b)o(y)75 1489 y(the)g(call.)20 b(The)13 b(group)f(ob)s(ject) g(is)h(actually)h(deallo)q(cated)g(only)f(if)g(there)g(are)f(no)g (other)h(activ)o(e)f(references)75 1545 y(to)j(it.)23 b(If)17 b(a)e(group)h(has)g(b)q(een)h(b)q(ound)g(to)f(one)g(or)g(more)f (comm)o(unicators,)h(then)g(the)g(user)h(has)e(to)h(free)75 1602 y(all)g(\\copies")g(of)e(the)i(group.)166 1737 y Fn(Implemen)o(tati)o(on)c(note:)166 1791 y Fq(One)18 b(can)f(k)o(eep)h(a)f(reference)i(coun)o(t)f(that)f(is)g(incremen)o (ted)h(for)e(eac)o(h)i(call)e(to)h Fl(MPI)p 1516 1791 13 2 v 15 w(COMM)p 1661 1791 V 14 w(MAKE)g Fq(and)75 1841 y Fl(MPI)p 152 1841 V 14 w(COMM)p 296 1841 V 15 w(DUP)p Fq(,)d(and)h(and)g(decremen)o(ted)i(for)e(eac)o(h)h(call)e(to)h Fl(MPI)p 1194 1841 V 15 w(GROUP)p 1351 1841 V 13 w(FREE)f Fq(or)h Fl(MPI)p 1606 1841 V 15 w(COMM)p 1751 1841 V 14 w(FREE)p Fq(;)75 1890 y(the)f(group)g(ob)r(ject)h(is)f(ultimately)d (deallo)q(cated)j(when)g(the)h(reference)h(coun)o(t)e(drops)h(to)f (zero.)75 2136 y Ft(5.4)59 b(Communicato)n(r)19 b(Management)75 2244 y FC(This)d(section)h(describ)q(es)g(the)f(manipulation)h(of)f (comm)o(unicators)f(in)i(MPI.)e(Op)q(erations)i(that)e(access)75 2300 y(comm)o(unicators)g(are)g(lo)q(cal)h(and)f(their)h(execution)g (do)q(es)g(not)f(require)h(in)o(terpro)q(cess)g(comm)o(unication.)75 2357 y(Op)q(erations)22 b(that)e(create)h(comm)o(unicators)g(are)g (collectiv)o(e)i(and)e(ma)o(y)f(require)i(in)o(terpro)q(cess)g(com-)75 2413 y(m)o(unication.)28 b(\(High)18 b(qualit)o(y)g(implemen)o(tations) g(will)h(b)q(e)g(able)f(to)f(amortize)g(these)h(o)o(v)o(erheads)f(o)o (v)o(er)75 2470 y(sev)o(eral)e(calls.\))75 2611 y Fp(5.4.1)49 b(Communicato)o(r)16 b(Accesso)o(rs)75 2704 y FC(The)f(follo)o(wing)h (are)f(all)h(lo)q(cal)h(op)q(erations.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 113 121 113 120 bop 75 -100 a Fs(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b FC(113)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(COMM)p 318 45 V 16 w(SIZE\(comm,)13 b(size\))117 122 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 198 y(OUT)108 b Fv(size)519 b Fq(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)g(of)27 b Fl(comm)12 b Fq(\(in)o(teger\))75 322 y Fo(int)23 b(MPI)p 245 322 15 2 v 17 w(Comm)p 358 322 V 17 w(size\(MPI)p 567 322 V 16 w(Comm)g(comm,)g(int)h(*size\))75 409 y(MPI)p 150 409 V 17 w(COMM)p 263 409 V 16 w(SIZE\(COMM,)f(SIZE,)g(IERROR\))170 465 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 599 y Fv(MPI)p 160 599 14 2 v 16 w(COMM)p 318 599 V 16 w(RANK\(comm,)13 b(rank\))117 676 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 751 y(OUT)108 b Fv(rank)504 b Fq(rank)12 b(of)f(the)h(calling)e(pro)q(cess)j(in)f(group)f(of)23 b Fl(comm)9 b Fq(\(in)o(teger\))75 876 y Fo(int)23 b(MPI)p 245 876 15 2 v 17 w(Comm)p 358 876 V 17 w(rank\(MPI)p 567 876 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 963 y(MPI)p 150 963 V 17 w(COMM)p 263 963 V 16 w(RANK\(COMM,)f(RANK,)g(IERROR\))170 1019 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 1153 y Fv(MPI)p 160 1153 14 2 v 16 w(COMM)p 318 1153 V 16 w(GROUP\(comm,)14 b(group\))117 1230 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 1305 y(OUT)108 b Fv(group)479 b Fq(group)14 b(corresp)q(onding)h(to)f Fl(comm)e Fq(\(handle\))75 1430 y Fo(int)23 b(MPI)p 245 1430 15 2 v 17 w(Comm)p 358 1430 V 17 w(group\(MPI)p 591 1430 V 16 w(Comm)g(comm,)g(MPI)p 941 1430 V 17 w(Group)g(*group\))75 1516 y(MPI)p 150 1516 V 17 w(COMM)p 263 1516 V 16 w(GROUP\(COMM,)g(GROUP,)g(IERROR\))170 1573 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)75 1725 y Fp(5.4.2)49 b(Communicato)o(r)16 b(Constructo)o(rs)75 1811 y FC(The)f(follo)o(wing) g(are)g(collectiv)o(e)h(functions)f(that)f(are)h(in)o(v)o(ok)o(ed)g(b)o (y)f(all)i(pro)q(cesses)f(in)g(the)g(group)f(asso)q(ci-)75 1867 y(ated)h(with)h Fv(comm)p FC(.)166 1924 y(Note)f(that)g(there)g (is)h(a)g(c)o(hic)o(k)o(en-and-egg)g(asp)q(ect)g(to)f(MPI)g(in)i(that)d (a)h(comm)o(unicator)h(is)g(needed)75 1980 y(to)21 b(create)h(a)f(new)h (comm)o(unicator.)39 b(The)22 b(base)f(comm)o(unicator)h(for)f(all)i (MPI)e(comm)o(unicators)g(is)75 2037 y(prede\014ned)c(outside)f(of)e (MPI,)h(and)h(is)f Fv(MPI)p 839 2037 14 2 v 16 w(COMM)p 997 2037 V 17 w(W)o(ORLD)g FC(\(de\014ned)h(ab)q(o)o(v)o(e\).)75 2141 y Fv(MPI)p 160 2141 V 16 w(COMM)p 318 2141 V 16 w(DUP\(comm,)d(new)o(comm\))117 2218 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 2293 y(OUT)108 b Fv(new)o(comm)397 b Fq(cop)o(y)14 b(of)f Fl(comm)f Fq(\(handle\))75 2418 y Fo(int)23 b(MPI)p 245 2418 15 2 v 17 w(Comm)p 358 2418 V 17 w(dup\(MPI)p 543 2418 V 16 w(Comm)g(comm,)h(MPI)p 894 2418 V 16 w(Comm)g(*newcomm\))75 2504 y(MPI)p 150 2504 V 17 w(COMM)p 263 2504 V 16 w(DUP\(COMM,)f (NEWCOMM,)g(IERROR\))170 2561 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)166 2647 y Fv(MPI)p 251 2647 14 2 v 16 w(COMM)p 409 2647 V 16 w(DUP)15 b FC(Duplicates)i(the)e(existing)h(comm)o(unicator)f Fv(comm)f FC(with)h(cac)o(hed)h(attributes)75 2704 y(copied)j(or)f(not) g(copied)i(\(according)e(to)g(the)g(cop)o(y-callbac)o(k)h(p)q(olicy)h (of)e(eac)o(h)g(individual)k(cac)o(hed)c(at-)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 114 122 114 121 bop 75 -100 a FC(114)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FC(tribute\).)35 b(Returns)21 b(in)g Fv(new)o(comm)e FC(a)h(new)g(comm)o(unicator)g(with)g(the)h(same)f(group,)h(same)e(cac) o(hed)75 102 y(information,)c(but)g(a)g(new)h(con)o(text.)166 241 y Fn(Implemen)o(tati)o(on)g(note:)49 b Fq(One)17 b(need)h(not)f(actually)g(cop)o(y)g(the)g(group)g(information,)e(but)i (only)f(add)75 298 y(a)j(new)h(reference)i(and)d(incremen)o(t)h(the)g (reference)i(coun)o(t.)35 b(Cop)o(y)19 b(on)g(write)h(can)f(b)q(e)h (used)h(for)e(the)h(cac)o(hed)75 354 y(information.)75 541 y Fv(MPI)p 160 541 14 2 v 16 w(COMM)p 318 541 V 16 w(MAKE\(comm,)13 b(group,)h(new)o(comm\))117 618 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 694 y(IN)155 b Fv(group)479 b Fq(Group,)14 b(whic)o(h)h(is)f(a)g (subset)i(of)e(the)h(group)g(of)e Fl(comm)g Fq(\(han-)905 751 y(dle\))117 827 y(OUT)108 b Fv(new)o(comm)397 b Fq(new)15 b(comm)o(uni)o(cator)d(\(handle\))75 952 y Fo(int)23 b(MPI)p 245 952 15 2 v 17 w(Comm)p 358 952 V 17 w(make\(MPI)p 567 952 V 16 w(Comm)g(comm,)g(MPI)p 917 952 V 17 w(Group)g(group,)g (MPI)p 1316 952 V 17 w(Comm)h(*newcomm\))75 1039 y(MPI)p 150 1039 V 17 w(COMM)p 263 1039 V 16 w(MAKE\(COMM,)f(GROUP,)g(NEWCOMM,) g(IERROR\))170 1095 y(INTEGER)g(COMM,)h(GROUP,)f(NEWCOMM,)f(IERROR)75 1182 y FC(This)14 b(function)g(creates)f(a)g(new)h(comm)o(unicator)f Fv(new)o(comm)f FC(with)i(comm)o(unication)g(group)f(de\014ned)i(b)o(y) 75 1238 y Fv(group)f FC(and)h(a)e(new)i(con)o(text.)k(No)14 b(cac)o(hed)h(information)f(propagates)f(from)h Fv(comm)e FC(to)i Fv(new)o(comm)p FC(.)k(The)75 1295 y(function)e(returns)f(a)g (n)o(ull)i(handle)f(to)f(pro)q(cesses)g(that)g(are)g(not)f(in)i Fv(group)p FC(.)166 1352 y(The)f(call)h(is)f(erroneous)g(if)g(not)f (all)i Fv(group)f FC(argumen)o(ts)f(ha)o(v)o(e)g(the)h(same)f(v)m (alue,)i(or)e(if)h Fv(group)g FC(is)g(not)75 1408 y(a)j(subset)h(of)f (the)h(group)f(asso)q(ciated)h(with)g Fv(comm)p FC(.)28 b(Note)18 b(that)g(the)g(call)i(is)f(to)f(b)q(e)h(executed)h(b)o(y)e (all)75 1465 y(pro)q(cesses)e(in)g Fv(comm)p FC(,)d(ev)o(en)i(if)h (they)f(do)g(not)g(b)q(elong)h(to)f(the)g(new)g(group.)166 1604 y Fn(Implemen)o(tati)o(on)e(note:)40 b Fq(Since)15 b(all)e(pro)q(cesses)18 b(calling)13 b Fl(MPI)p 1215 1604 13 2 v 14 w(COMM)p 1359 1604 V 15 w(DUP)h Fq(or)g Fl(MPI)p 1600 1604 V 15 w(COMM)p 1745 1604 V 14 w(MAKE)75 1661 y Fq(pro)o(vide)d(the)g(same)f Fl(group)h Fq(argumen)o(t,)f(it)g (is)h(theoretically)g(p)q(ossible)g(to)f(agree)i(on)e(a)h(group-wide)f (unique)h(con)o(text)75 1717 y(with)h(no)h(comm)o(unication.)h(Ho)o(w)o (ev)o(er,)f(lo)q(cal)f(execution)i(of)e(these)i(functions)f(requires)h (use)g(of)e(a)g(larger)h(con)o(text)75 1774 y(name)h(space)i(and)e (reduces)j(error-c)o(hec)o(king.)22 b(Implemen)o(tations)12 b(ma)o(y)h(strik)o(e)i(v)n(arious)g(compromises)e(b)q(et)o(w)o(een)75 1830 y(these)k(con\015icting)f(goals.)23 b(If)16 b(new)g(comm)o (unicators)e(are)i(created)i(without)d(sync)o(hronizing)h(the)h(pro)q (cesses)i(in-)75 1887 y(v)o(olv)o(ed)d(then)i(the)g(comm)o(ucatio)o(n)d (system)h(should)h(b)q(e)h(able)f(to)f(cop)q(e)i(with)f(messages)g (arriving)f(in)h(a)f(con)o(text)75 1943 y(that)e(has)g(not)g(y)o(et)g (b)q(een)h(allo)q(cated)e(at)h(the)h(receiving)f(pro)q(cess.)75 2130 y Fv(MPI)p 160 2130 14 2 v 16 w(COMM)p 318 2130 V 16 w(SPLIT\(comm,)e(colo)o(r,)i(k)o(ey)l(,)h(new)o(comm\))117 2207 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 2283 y(IN)155 b Fv(colo)o(r)495 b Fq(con)o(trol)14 b(of)f(subset)i(assignmen)o(t)e(\(in)o(teger\))117 2359 y(IN)155 b Fv(k)o(ey)525 b Fq(con)o(trol)14 b(of)f(rank)h(assigmen)o(t) f(\(in)o(teger\))117 2435 y(OUT)108 b Fv(new)o(comm)397 b Fq(new)15 b(comm)o(uni)o(cator)d(\(handle\))75 2560 y Fo(int)23 b(MPI)p 245 2560 15 2 v 17 w(Comm)p 358 2560 V 17 w(split\(MPI)p 591 2560 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g (key,)h(MPI)p 1419 2560 V 16 w(Comm)g(*newcomm\))75 2647 y(MPI)p 150 2647 V 17 w(COMM)p 263 2647 V 16 w(SPLIT\(COMM,)f(COLOR,)g (KEY,)g(NEWCOMM,)g(IERROR\))170 2704 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,) g(NEWCOMM,)g(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 115 123 115 122 bop 75 -100 a Fs(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FC(115)75 45 y(This)19 b(function)f(partitions)g(the)h(group)e(asso)q (ciated)h(with)h Fv(comm)d FC(in)o(to)i(disjoin)o(t)g(subgroups,)h(one) f(for)75 102 y(eac)o(h)g(v)m(alue)g(of)f Fv(colo)o(r)p FC(.)26 b(Eac)o(h)17 b(subgroup)h(con)o(tains)f(all)i(pro)q(cesses)f (of)f(the)g(same)g(color.)27 b(Within)19 b(eac)o(h)75 158 y(subgroup,)24 b(the)f(pro)q(cesses)g(are)f(rank)o(ed)h(in)g(the)g (order)g(de\014ned)h(b)o(y)e(the)h(v)m(alue)h(of)e(the)h(argumen)o(t)75 214 y Fv(k)o(ey)p FC(,)d(with)g(ties)g(brok)o(en)g(according)g(to)f (their)h(rank)f(in)i(the)f(old)g(group.)33 b(A)19 b(new)h(comm)o (unicator)f(is)75 271 y(created)14 b(for)f(eac)o(h)h(subgroup)g(and)g (returned)g(in)g Fv(new)o(comm)p FC(.)k(A)c(pro)q(cess)g(ma)o(y)f (supply)i(the)f(color)g(v)m(alue)75 327 y Fv(MPI)p 160 327 14 2 v 16 w(UNDEFINED)p FC(,)i(in)h(whic)o(h)h(case)f Fv(new)o(comm)e FC(returns)h(a)g(n)o(ull)i(comm)o(unicator)f(handle.)25 b(This)17 b(is)g(a)75 384 y(collectiv)o(e)g(call,)f(but)f(eac)o(h)h (pro)q(cess)f(can)g(pro)o(vide)h(di\013eren)o(t)f(v)m(alues)i(for)d Fv(colo)o(r)g FC(and)i Fv(k)o(ey)p FC(.)166 440 y(A)27 b(call)i(to)d Fv(MPI)p 475 440 V 16 w(COMM)p 633 440 V 17 w(MAKE\(comm,)h(group,)j(new)o(comm\))c FC(is)h(equiv)m(alen)o(t)i (to)e(a)g(call)i(to)75 497 y Fv(MPI)p 160 497 V 16 w(COMM)p 318 497 V 16 w(SPLIT\(comm,)8 b(colo)o(r,)i(k)o(ey)l(,)h(new)o(comm\))p FC(,)e(where)h(all)h(mem)o(b)q(ers)g(of)e Fv(group)h FC(pro)o(vide)h Fv(colo)o(r)e FC(=)23 b(0)75 553 y(and)15 b Fv(k)o(ey)f FC(=)29 b(rank)15 b(in)g Fv(group)p FC(,)f(and)h(all)g (pro)q(cesses)g(that)f(are)g(not)g(mem)o(b)q(ers)h(of)f Fv(group)g FC(pro)o(vide)h Fv(colo)o(r)e FC(=)75 610 y Fv(MPI)p 160 610 V 16 w(UNDEFINED)p FC(.)k(The)i(function)f Fv(MPI)p 819 610 V 16 w(COMM)p 977 610 V 17 w(SPLIT)f FC(allo)o(ws)h(more)f(general)i(partitioning)g(of)e(a)75 666 y(group)e(in)o(to)g(one)g(or)g(more)g(subgroups)g(with)h(optional)f (reordering.)75 788 y Fp(5.4.3)49 b(Communicato)o(r)16 b(Destructo)o(rs)75 922 y Fv(MPI)p 160 922 V 16 w(COMM)p 318 922 V 16 w(FREE\(comm\))117 999 y Fq(INOUT)62 b Fv(comm)470 b Fq(comm)o(unicator)11 b(to)j(b)q(e)h(destro)o(y)o(ed)g(\(handle\))75 1123 y Fo(int)23 b(MPI)p 245 1123 15 2 v 17 w(Comm)p 358 1123 V 17 w(free\(MPI)p 567 1123 V 16 w(Comm)g(*comm\))75 1210 y(MPI)p 150 1210 V 17 w(COMM)p 263 1210 V 16 w(FREE\(COMM,)g (IERROR\))170 1266 y(INTEGER)g(COMM,)h(IERROR)166 1353 y FC(This)13 b(collectiv)o(e)g(op)q(eration)g(marks)e(the)h(comm)o (unication)h(ob)s(ject)f(for)f(deallo)q(cation.)20 b(The)13 b(handle)75 1409 y(is)i(set)g(to)f(n)o(ull.)21 b(The)15 b(ob)s(ject)f(is)h(actually)g(deallo)q(cated)h(only)g(if)f(there)f(are) h(no)f(other)h(activ)o(e)f(references)75 1466 y(to)h(it.)21 b(The)15 b(asso)q(ciated)h(group)f(ma)o(y)g(also)g(b)q(e)h(deallo)q (cated,)h(if)f(there)f(are)g(no)h(activ)o(e)f(references)h(to)f(it.)75 1522 y(It)g(is)h(the)f(user)h(resp)q(onsibilit)o(y)h(to)e(free)g(eac)o (h)g(cop)o(y)g(of)g(a)g(comm)o(unicator)g(ob)s(ject.)166 1655 y Fn(Implemen)o(tati)o(on)g(note:)47 b Fq(A)17 b(reference)i(coun) o(t)e(mec)o(hanism)d(can)j(b)q(e)g(used:)24 b(the)17 b(reference)i(coun)o(t)e(is)75 1705 y(incremen)o(ted)d(b)o(y)g(eac)o(h) g(call)f(to)g Fl(MPI)p 663 1705 13 2 v 15 w(COMM)p 808 1705 V 14 w(DUP)p Fq(,)g(and)g(decremen)o(ted)i(b)o(y)f(eac)o(h)g(call) f(to)g Fl(MPI)p 1606 1705 V 15 w(COMM)p 1751 1705 V 14 w(FREE)p Fq(.)75 1754 y(The)h(ob)r(ject)h(is)f(ultimately)d(deallo)q (cated)j(when)h(the)f(coun)o(t)g(reac)o(hes)i(zero.)166 1811 y(Though)k(collectiv)o(e,)i(it)e(is)h(an)o(ticipated)f(that)h (this)f(op)q(eration)h(will)e(normally)f(b)q(e)j(lo)q(cal,)g(though)f (the)75 1867 y(debugging)13 b(v)o(ersion)h(of)g(an)f(MPI)h(library)g (migh)o(t)d(c)o(ho)q(ose)k(to)f(sync)o(hronize.)75 2094 y Ft(5.5)59 b(Inter-Communication)75 2195 y FC(This)18 b(section)h(in)o(tro)q(duces)f(the)g(concept)g(of)f(in)o(ter-comm)o (unication)i(and)f(describ)q(es)i(the)d(p)q(ortions)h(of)75 2252 y(MPI)j(that)g(supp)q(ort)g(it.)38 b(It)22 b(describ)q(es)h(supp)q (ort)e(for)g(writing)g(programs)f(that)h(con)o(tain)g(user-lev)o(el)75 2308 y(serv)o(ers.)h(It)16 b(also)f(describ)q(es)j(a)e(name)g(service)g (that)g(simpli\014es)i(writing)e(programs)f(con)o(taining)i(in)o(ter-) 75 2365 y(comm)o(unication.)166 2421 y(All)h(p)q(oin)o(t-to-p)q(oin)o (t)g(comm)o(unication)g(describ)q(ed)h(th)o(us)d(far)h(has)g(in)o(v)o (olv)o(ed)g(comm)o(unication)h(b)q(e-)75 2478 y(t)o(w)o(een)c(pro)q (cesses)g(that)f(are)h(mem)o(b)q(ers)g(of)f(the)h(same)g(group.)19 b(This)c(t)o(yp)q(e)f(of)f(comm)o(unication)i(is)f(called)75 2534 y(\\in)o(tra-comm)o(unication")h(and)h(the)f(comm)o(unicator)g (used)h(is)f(called)i(an)e(\\in)o(tra-comm)o(unicator.")166 2591 y(In)c(mo)q(dular)g(and)f(m)o(ulti-discipl)q(inary)j (applications,)g(di\013eren)o(t)d(pro)q(cess)h(groups)f(execute)h (distinct)75 2647 y(mo)q(dules)i(and)f(pro)q(cesses)h(within)g (di\013eren)o(t)f(mo)q(dules)h(comm)o(unicate)f(with)h(one)f(another)f (in)i(a)f(pip)q(eline)75 2704 y(or)j(a)f(more)h(general)h(mo)q(dule)g (graph.)j(In)d(these)f(applications,)i(the)e(most)f(natural)h(w)o(a)o (y)f(for)h(a)g(pro)q(cess)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 116 124 116 123 bop 75 -100 a FC(116)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FC(to)g(sp)q(ecify)i(a)e(target)f(pro)q(cess)i(is)g(b)o(y)g(the)g (rank)f(of)g(the)h(target)e(pro)q(cess)i(within)h(the)e(target)g (group.)20 b(In)75 102 y(applications)13 b(that)e(con)o(tain)h(in)o (ternal)h(user-lev)o(el)g(serv)o(ers,)f(eac)o(h)f(serv)o(er)h(ma)o(y)f (b)q(e)h(a)f(pro)q(cess)h(group)g(that)75 158 y(pro)o(vides)17 b(services)f(to)g(one)g(or)g(more)f(clien)o(ts,)i(and)g(eac)o(h)f (clien)o(t)h(ma)o(y)f(b)q(e)g(a)g(pro)q(cess)h(group)e(that)h(uses)75 214 y(the)h(services)h(of)f(one)g(or)g(more)g(serv)o(ers.)25 b(It)17 b(is)h(again)f(most)f(natural)i(to)e(sp)q(ecify)j(the)e(target) f(pro)q(cess)75 271 y(b)o(y)d(rank)g(within)i(the)e(target)f(group)i (in)g(these)f(applications.)21 b(This)14 b(t)o(yp)q(e)g(of)e(comm)o (unication)j(is)e(called)75 327 y(\\in)o(ter-comm)o(unication")j(and)f (the)g(comm)o(unicator)g(used)h(is)g(called)h(an)e(\\in)o(ter-comm)o (unicator.")166 385 y(An)23 b(in)o(ter-comm)o(unication)g(op)q(eration) g(is)f(a)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unication)g(b)q(et)o(w)o (een)g(pro-)75 441 y(cesses)16 b(in)h(di\013eren)o(t)f(groups.)21 b(The)16 b(group)f(con)o(taining)h(a)g(pro)q(cess)g(that)f(initiates)i (an)e(in)o(ter-comm)o(un-)75 498 y(ication)i(op)q(eration)f(is)g (called)h(the)f(\\lo)q(cal)h(group,")e(that)g(is,)h(the)g(sender)h(in)f (a)g(send)g(and)g(the)g(receiv)o(er)75 554 y(in)g(a)e(receiv)o(e.)21 b(The)15 b(group)g(con)o(taining)g(the)g(target)f(pro)q(cess)h(is)g (called)i(the)e(\\remote)f(group,")g(that)g(is,)75 610 y(the)j(receiv)o(er)i(in)f(a)f(send)h(and)f(the)h(sender)g(in)g(a)f (receiv)o(e.)27 b(As)17 b(in)i(in)o(tra-comm)o(unication,)f(the)f (target)75 667 y(pro)q(cess)12 b(is)g(sp)q(eci\014ed)i(using)e(a)f Fv(\(communicato)o(r,)f(rank\))i FC(pair.)19 b(Unlik)o(e)13 b(in)o(tra-comm)o(unication,)g(the)e(rank)75 723 y(is)16 b(relativ)o(e)f(to)g(a)g(second,)g(remote)g(group.)166 781 y(Lo)q(cal)g(and)g(remote)e(groups)h(of)g(an)g(in)o(ter-comm)o (unicator)h(are)f(not)f(required)j(to)d(b)q(e)i(disjoin)o(t.)21 b(An)75 837 y(in)o(tra-comm)o(unicator)14 b(can)h(b)q(e)h(seen)f(as)f (a)g(particular)i(case)e(of)h(an)f(in)o(ter-comm)o(unicator,)h(where)g (lo)q(cal)75 893 y(and)h(remote)g(groups)f(happ)q(en)j(to)d(b)q(e)i (iden)o(tical.)24 b(The)16 b(seman)o(tics)h(of)e(accessor)h(function)h (are)f(de\014ned)75 950 y(so)k(as)g(to)g(b)q(e)h(consisten)o(t)g(with)g (this)g(in)o(terpretation.)36 b(Here)20 b(is)h(a)g(summary)f(of)g(the)g (prop)q(erties)h(of)75 1006 y(in)o(ter-comm)o(unication)16 b(and)g(in)o(ter-comm)o(unicators:)143 1103 y FA(\017)23 b FC(The)c(syn)o(tax)g(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)g(is)g(the)f(same)g(for)g(b)q(oth)h(in)o(ter-)f(and)h(in)o (tra-)189 1160 y(comm)o(unication.)26 b(The)17 b(same)g(comm)o (unicator)g(can)g(b)q(e)g(used)h(b)q(oth)f(for)g(send)g(and)h(for)e (receiv)o(e)189 1216 y(op)q(erations.)143 1313 y FA(\017)23 b FC(A)14 b(target)g(pro)q(cess)h(is)g(addressed)h(b)o(y)f(its)g(rank)f (in)i(the)f(remote)f(group,)g(b)q(oth)h(for)f(sends)i(and)f(for)189 1370 y(receiv)o(es.)143 1467 y FA(\017)23 b FC(Comm)o(unications)12 b(using)i(an)e(in)o(ter-comm)o(unicator)h(are)f(guaran)o(teed)g(not)g (to)g(con\015ict)h(with)g(an)o(y)189 1523 y(comm)o(unications)i(that)g (use)h(a)e(di\013eren)o(t)i(comm)o(unicator.)143 1620 y FA(\017)23 b FC(An)15 b(in)o(ter-comm)o(unicator)g(cannot)g(b)q(e)h (used)g(for)f(collectiv)o(e)i(comm)o(unication.)143 1717 y FA(\017)23 b FC(A)15 b(comm)o(unicator)g(will)i(pro)o(vide)e(either)h (in)o(tra-)f(or)g(in)o(ter-comm)o(unication,)h(nev)o(er)f(b)q(oth.)75 1814 y(The)i(routine)f Fv(MPI)p 412 1814 14 2 v 16 w(COMM)p 570 1814 V 17 w(TEST)p 701 1814 V 16 w(INTER)h FC(ma)o(y)e(b)q(e)i (used)g(to)f(determine)h(if)g(a)f(comm)o(unicator)g(is)h(an)75 1870 y(in)o(ter-)i(or)e(in)o(tra-comm)o(unicator.)29 b(In)o(ter-comm)o(unicators)18 b(can)g(b)q(e)h(used)g(as)f(argumen)o (ts)f(to)g(some)h(of)75 1927 y(the)f(other)f(comm)o(unicator)h(access)g (routines.)25 b(In)o(ter-comm)o(unicators)17 b(cannot)f(b)q(e)i(used)f (as)g(input)h(to)75 1983 y(an)o(y)d(of)g(the)g(constructor)f(routines)i (for)f(in)o(tra-comm)o(unicators.)166 2117 y Fn(Implemen)o(tati)o(on)e (note:)38 b Fq(Comm)o(unicators)12 b(can)i(b)q(e)h(represen)o(ted)j(in) c(eac)o(h)h(pro)q(cess)h(b)o(y)e(a)g(tuple)h(con-)75 2166 y(sisting)f(of:)75 2243 y Fn(group)75 2321 y(send)p 173 2321 15 2 v 16 w(con)o(text)75 2399 y(receiv)o(e)p 223 2399 V 16 w(con)o(text)75 2477 y(source)75 2554 y Fq(F)m(or)f(in)o(ter-comm)o(unicators,)e Fn(group)g Fq(describ)q(es)16 b(the)e(remote)f(group,)g(and)g Fn(source)f Fq(is)h(the)h(rank)f(of)g (the)h(pro)q(cess)75 2604 y(in)j(the)i(lo)q(cal)e(group.)30 b(F)m(or)17 b(in)o(tra-comm)o(unicators,)f Fn(group)g Fq(is)i(the)g(comm)o(unicator)d(group)j(\(remote=lo)q(cal\),)75 2654 y Fn(source)h Fq(is)h(the)g(rank)g(of)f(the)i(pro)q(cess)h(in)d (this)h(group,)h(and)f Fn(send)h(con)o(text)e Fq(and)h Fn(receiv)o(e)h(con)o(text)e Fq(are)75 2704 y(iden)o(tical.)e(A)d (group)g(is)g(represen)o(ted)i(b)o(y)e(a)g(rank-to-absolute-address)g (translation)g(table.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 117 125 117 124 bop 75 -100 a Fs(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FC(117)166 45 y Fq(The)12 b(in)o(ter-comm)o(unicator)c(cannot)j(b)q (e)h(discussed)h(sensibly)e(without)g(considering)h(pro)q(cesses)h(in)e (b)q(oth)g(the)75 95 y(lo)q(cal)k(and)g(remote)h(groups.)23 b(Imagine)14 b(a)i(pro)q(cess)h Fn(P)e Fq(in)h(group)f Fc(P)s Fq(,)h(whic)o(h)g(has)f(an)h(in)o(ter-comm)o(unicator)d Fn(C)1835 101 y Fb(P)1863 95 y Fq(,)75 145 y(and)j(a)h(pro)q(cess)h Fn(Q)e Fq(in)g(group)h Fc(Q)p Fq(,)f(whic)o(h)g(has)h(an)f(in)o (ter-comm)o(unicator)e Fn(C)1293 151 y Fb(Q)1323 145 y Fq(.)26 b(\(Note)17 b(that)f Fc(P)k Fq(and)c Fc(Q)h Fq(do)f(not)75 195 y(ha)o(v)o(e)e(to)f(b)q(e)i(distinct.\))j(Then)145 266 y Fc(\017)23 b Fn(C)223 272 y Fb(P)251 266 y Fn(.group)13 b Fq(describ)q(es)j(the)e(group)g Fc(Q)g Fq(and)f Fn(C)927 272 y Fb(Q)957 266 y Fn(.group)g Fq(describ)q(es)j(the)e(group)g Fc(P)s Fq(.)145 338 y Fc(\017)23 b Fn(C)223 344 y Fb(P)251 338 y Fn(.send)p 362 338 15 2 v 16 w(con)o(text)15 b(=)h Fq(C)629 344 y Fb(Q)658 338 y Fn(.receiv)o(e)p 819 338 V 16 w(con)o(text)c Fq(and)i(the)h(con)o(text)f(is)g(unique)g(in)f Fc(Q)p Fq(;)189 388 y Fn(C)223 394 y Fb(P)251 388 y Fn(.receiv)o(e)p 412 388 V 16 w(con)o(text)i(=)32 b(C)699 394 y Fb(Q)728 388 y Fn(.send)p 839 388 V 17 w(con)o(text)12 b Fq(and)i(this)g(con)o (text)g(is)g(unique)g(in)f Fc(P)s Fq(.)145 460 y Fc(\017)23 b Fn(C)223 466 y Fb(P)251 460 y Fn(.source)13 b Fq(is)h(rank)g(of)f Fn(P)h Fq(in)f Fc(P)k Fq(and)d Fn(C)851 466 y Fb(Q)881 460 y Fn(.source)f Fq(is)g(rank)h(of)f Fn(Q)h Fq(in)g Fc(Q)p Fq(.)166 532 y(Assume)g(that)h Fn(P)f Fq(sends)i(a)f(message)f (to)g Fn(Q)h Fq(using)f(the)i(in)o(ter-comm)o(unicator.)h(Then)e Fn(P)f Fq(uses)i(the)g Fn(group)75 582 y Fq(table)e(to)g(\014nd)g(the)g (absolute)g(address)h(of)e Fn(Q)p Fq(;)h Fn(source)e Fq(and)i Fn(send)p 1129 582 V 16 w(con)o(text)e Fq(are)j(app)q(ended)g (to)e(the)i(message.)166 632 y(Assume)d(that)g Fn(Q)g Fq(p)q(osts)h(a)f(receiv)o(e)i(with)e(an)g(explicit)g(source)h(argumen) o(t)e(using)h(the)h(in)o(ter-comm)o(unicator.)75 681 y(Then)19 b Fn(Q)f Fq(matc)o(hes)h Fn(receiv)o(e)p 556 681 V 16 w(con)o(text)e Fq(to)h(the)h(message)g(con)o(text)g(and)f (source)i(argumen)o(t)e(to)g(the)h(message)75 731 y(source.)166 788 y(The)14 b(same)f(algorithm)e(is)j(appropriate)g(for)g(in)o (tra-comm)n(unicators)e(as)i(w)o(ell.)75 991 y Fp(5.5.1)49 b(Communicato)o(r)16 b(Accesso)o(rs)75 1124 y Fv(MPI)p 160 1124 14 2 v 16 w(COMM)p 318 1124 V 16 w(TEST)p 448 1124 V 17 w(INTER\(comm,)d(\015ag\))117 1201 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))117 1275 y(OUT)108 b Fv(\015ag)518 b Fq(\(logical\))75 1399 y Fo(int)23 b(MPI)p 245 1399 15 2 v 17 w(Comm)p 358 1399 V 17 w(test)p 471 1399 V 16 w(inter\(MPI)p 703 1399 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 1486 y(MPI)p 150 1486 V 17 w(COMM)p 263 1486 V 16 w(TEST)p 375 1486 V 17 w(INTER\(COMM,)f(FLAG,)i(IERROR\))170 1542 y(INTEGER)f(COMM,)h (IERROR)170 1599 y(LOGICAL)f(FLAG)75 1685 y FC(This)d(lo)q(cal)g (routine)g(allo)o(ws)f(the)g(calling)i(pro)q(cess)f(to)e(determine)i (if)g(a)f(comm)o(unicator)g(is)g(an)h(in)o(ter-)75 1741 y(comm)o(unicator)15 b(or)g(an)g(in)o(tra-comm)o(unicator.)k(It)c (returns)g Fl(true)i FC(if)e(it)h(is)g(an)f(in)o(tercomm)o(unicator.) 166 1798 y(When)j(an)f(in)o(ter-comm)o(unicator)h(is)g(used)g(as)f(an)g (input)i(argumen)o(t)d(to)h(the)h(comm)o(unicator)f(ac-)75 1854 y(cessors)e(describ)q(ed)i(under)f(in)o(tra-comm)o(unication,)f (the)h(follo)o(wing)g(table)f(describ)q(es)i(b)q(eha)o(vior.)p 270 1939 1410 2 v 269 1995 2 57 v 652 1978 a Fv(MPI)p 737 1978 14 2 v 16 w(COMM)p 895 1978 V 17 w(*)d FC(F)l(unction)j(Beha)o (vior)p 1679 1995 2 57 v 269 2052 V 655 2035 a(\(in)f(In)o(ter-Comm)o (unication)f(Mo)q(de\))p 1679 2052 V 270 2053 1410 2 v 270 2063 V 269 2120 2 57 v 295 2103 a Fv(MPI)p 380 2103 14 2 v 16 w(COMM)p 538 2103 V 16 w(SIZE)p 729 2120 2 57 v 111 w FC(returns)g(the)g(size)h(of)f(the)g(remote)g(group.)p 1679 2120 V 269 2176 V 295 2159 a Fv(MPI)p 380 2159 14 2 v 16 w(COMM)p 538 2159 V 16 w(GROUP)p 729 2176 2 57 v 52 w FC(returns)g(the)g(remote)g(group.)p 1679 2176 V 269 2233 V 295 2216 a Fv(MPI)p 380 2216 14 2 v 16 w(COMM)p 538 2216 V 16 w(RANK)p 729 2233 2 57 v 81 w FC(returns)g(the)g(rank)g (in)h(the)f(lo)q(cal)i(group)p 1679 2233 V 270 2234 1410 2 v 75 2336 a Fp(5.5.2)49 b(Intercommunicato)o(r)14 b(Constructo)o(rs)h (and)h(Destructo)o(rs)75 2421 y FC(The)f(functions)h Fv(MPI)p 449 2421 14 2 v 16 w(COMM)p 607 2421 V 16 w(DUP)p FC(,)f Fv(MPI)p 826 2421 V 15 w(COMM)p 983 2421 V 17 w(MAKE)f FC(and)h Fv(MPI)p 1313 2421 V 16 w(COMM)p 1471 2421 V 17 w(SPLIT)f FC(do)h(not)f(apply)75 2478 y(to)c(in)o(tercomm)o (unicators.)18 b(In)10 b(addition,)i(the)e(functions)h Fv(MPI)p 1137 2478 V 16 w(INTERCOMM)p 1427 2478 V 17 w(ST)l(ART)g FC(and)f Fv(MPI)p 1762 2478 V 16 w(INTERCOMM)p 2052 2478 V 17 w(FINISH)75 2534 y FC(can)g(b)q(e)h(used)g(to)e(bind)j (t)o(w)o(o)d(groups)g(in)o(to)i(an)f(in)o(tercomm)o(unicator;)h(the)f (function)h Fv(MPI)p 1555 2534 V 16 w(INTERCOMM)p 1845 2534 V 17 w(MERGE)75 2591 y FC(creates)k(an)g(in)o(tracomm)o(unicator)g (b)o(y)g(merging)g(the)g(t)o(w)o(o)f(groups)h(of)g(an)g(in)o(tercomm)o (unicator.)166 2647 y(Construction)d(of)g(an)h(in)o(ter-comm)o (unicator)f(from)g(t)o(w)o(o)f(in)o(tracomm)o(unicators)h(requires)h (separate)75 2704 y(collectiv)o(e)k(op)q(erations)f(in)g(the)f(lo)q (cal)i(group)e(and)h(in)g(the)f(remote)g(group,)g(b)q(oth)h(for)e(the)i (starting)f(and)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 118 126 118 125 bop 75 -100 a FC(118)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FC(the)k(\014nishing)i(op)q(erations,)e(as)g(w)o(ell)g(as)g(a)f(p) q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)f(b)q(et)o(w)o(een)h(a)e (pro)q(cess)h(in)75 102 y(the)c(lo)q(cal)i(group)e(and)g(a)g(pro)q (cess)h(in)g(the)f(remote)g(group.)20 b(The)c(explicit)h(sync)o (hronization)f(can)g(cause)75 158 y(deadlo)q(c)o(k)f(in)h(mo)q(dular)f (programs)e(with)i(cyclic)h(comm)o(unication)g(graphs,)e(ev)o(en)g(if)h (within)h(eac)o(h)f(lo)q(cal)75 214 y(group)g(calls)h(are)f(executed)h (in)g(the)g(same)e(order,)166 271 y Fv(MPI)p 251 271 14 2 v 16 w(INTERCOMM)p 541 271 V 16 w(ST)l(ART)21 b FC(and)e Fv(MPI)p 894 271 V 16 w(INTERCOMM)p 1184 271 V 17 w(FINISH)g FC(are)f(b)q(oth)i(collectiv)o(e)h(op)q(era-)75 327 y(tions)14 b(in)g(their)g(lo)q(cal)h(group,)e(but)h(the)g(call)g (within)h(eac)o(h)f(group)f(need)i(not)e(b)q(e)h(sync)o(hronized)h (with)f(the)75 384 y(call)i(in)g(the)g(other)e(group.)166 440 y(The)g(in)o(ter-comm)o(unicator)h(ob)s(jects)f(are)g(destro)o(y)o (ed)f(in)j(the)e(same)g(w)o(a)o(y)f(as)h(in)o(tra-comm)o(unicator)75 497 y(ob)s(jects,)g(that)h(is,)g(b)o(y)g(calling)i Fv(MPI)p 692 497 V 16 w(COMM)p 850 497 V 16 w(FREE)p FC(.)166 553 y(The)f(functions)h Fv(MPI)p 542 553 V 16 w(INTERCOMM)p 832 553 V 17 w(ST)l(ART)g FC(and)f Fv(MPI)p 1179 553 V 16 w(INTERCOMM)p 1469 553 V 16 w(FINISH)g FC(can)h(b)q(e)f(used)75 610 y(to)21 b(create)g(an)h(in)o(tercomm)o(unicator)g(from)f(t)o(w)o(o) f(existing)j(in)o(tracomm)o(unicators,)f(in)g(the)g(follo)o(wing)75 666 y(situation:)33 b(A)o(t)21 b(least)h(one)f(selected)i(mem)o(b)q(er) f(from)f(eac)o(h)g(group)g(\(the)h(\\group)f(leader"\))g(has)h(the)75 723 y(abilit)o(y)f(to)d(comm)o(unicate)i(with)g(the)f(selected)i(mem)o (b)q(er)f(from)e(the)i(other)f(group;)i(i.e.)33 b(a)19 b(\\paren)o(t")75 779 y(comm)o(unicator)h(exists)h(where)g(b)q(oth)f (leaders)h(b)q(elong,)i(and)d(eac)o(h)h(leader)g(kno)o(ws)f(the)h(rank) f(of)g(the)75 835 y(other)h(leader)i(in)f(this)g(paren)o(t)f(comm)o (unicator)h(\(the)f(t)o(w)o(o)f(leaders)i(could)h(b)q(e)f(the)g(same)f (pro)q(cess\).)75 892 y(F)l(urthermore,)14 b(mem)o(b)q(ers)i(of)e(eac)o (h)i(group)f(kno)o(w)f(the)h(rank)g(of)g(their)h(leader.)166 948 y(In)d(static)g(MPI)g(implemen)o(tation)h(the)f Fv(MPI)p 927 948 V 16 w(COMM)p 1085 948 V 16 w(W)o(ORLD)g FC(comm)o(unicator)f (can)h(b)q(e)h(this)f(par-)75 1005 y(en)o(t)k(comm)o(unicator.)24 b(Since)18 b(this)g(paren)o(t)e(comm)o(unicator)g(ma)o(y)h(b)q(e)g (shared)g(for)f(m)o(ultiple)j(in)o(tercom-)75 1061 y(m)o(unicator)e (creating)h(functions,)g(and)g(the)f(same)g(pro)q(cess)h(ma)o(y)f(b)q (e)h(leaders)g(for)f(sev)o(eral)h(suc)o(h)f(calls,)75 1118 y(an)e(additional)i(user-pro)o(vided)f(tag)e(is)i(used)g(to)e (di\013eren)o(tiate)i(calls.)166 1174 y(In)23 b(dynamic)g(MPI)g (implemen)o(tations,)i(where,)f(e.g.,)f(a)g(pro)q(cess)f(ma)o(y)g(spa)o (wn)g(new)h(c)o(hildren)75 1231 y(pro)q(cesses)18 b(during)h(an)f(MPI)f (execution,)j(the)e(paren)o(t)f(pro)q(cess)h(ma)o(y)f(b)q(e)i(the)f (\\bridge")g(b)q(et)o(w)o(een)g(the)75 1287 y(old)i(comm)o(unication)h (univ)o(erse)f(and)g(the)g(new)g(comm)o(unication)g(w)o(orld)g(that)f (includes)j(the)d(paren)o(t)75 1344 y(and)c(its)h(c)o(hildren.)75 1447 y Fv(MPI)p 160 1447 V 16 w(INTERCOMM)p 450 1447 V 16 w(ST)l(ART\(lo)q(cal)p 714 1447 V 17 w(comm,)53 b(lo)q(cal)p 1002 1447 V 16 w(leader,)i(p)q(eer)p 1278 1447 V 17 w(comm,)d(remote)p 1609 1447 V 16 w(leader,)i(tag,)75 1504 y(inter)p 164 1504 V 17 w(request\))117 1581 y Fq(IN)155 b Fv(lo)q(cal)p 409 1581 V 17 w(comm)365 b Fq(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e(\(handle\))117 1656 y(IN)155 b Fv(lo)q(cal)p 409 1656 V 17 w(leader)370 b Fq(rank)14 b(of)f(lo)q(cal)g(group)h(leader)g(in)g Fl(lo)q(cal)p 1517 1656 13 2 v 15 w(comm)e Fq(\(in)o(teger\))117 1730 y(IN)155 b Fv(p)q(eer)p 401 1730 14 2 v 18 w(comm)372 b Fq(\\paren)o(t")17 b(in)o(tra-comm)o(unicator;)e(signi\014can)o(t)i (only)f(at)h(the)905 1787 y Fl(lo)q(cal)p 987 1787 13 2 v 15 w(leader)f Fq(\(handle\))117 1862 y(IN)155 b Fv(remote)p 453 1862 14 2 v 16 w(leader)327 b Fq(rank)12 b(of)f(remote)g(group)h (leader)g(in)23 b Fl(p)q(eer)p 1548 1862 13 2 v 16 w(comm)p Fq(;)10 b(signi\014can)o(t)905 1918 y(only)j(at)h(the)h Fl(lo)q(cal)p 1201 1918 V 15 w(leader)g Fq(\(in)o(teger\))117 1993 y(IN)155 b Fv(tag)526 b Fq(\\safe")14 b(tag)f(\(in)o(teger\))117 2067 y(OUT)108 b Fv(inter)p 407 2067 14 2 v 17 w(request)349 b Fq(handle)14 b(for)27 b Fl(MPI)p 1193 2067 13 2 v 15 w(INTERCOMM)p 1459 2067 V 14 w(FINISH\(\))75 2192 y Fo(int)c(MPI)p 245 2192 15 2 v 17 w(Intercomm)p 478 2192 V 16 w(start\(MPI)p 710 2192 V 16 w(Comm)g(local)p 965 2192 V 17 w(comm,)g(int)h(local)p 1341 2192 V 16 w(leader,)393 2248 y(MPI)p 468 2248 V 17 w(Comm)f(peer)p 700 2248 V 17 w(comm,)g(int)h(remote)p 1100 2248 V 16 w(leader,)f(int)g(tag,)393 2305 y(MPI)p 468 2305 V 17 w(Intercomm)p 701 2305 V 16 w(request)g(*inter)p 1052 2305 V 16 w(request\))75 2391 y(MPI)p 150 2391 V 17 w(INTERCOMM)p 383 2391 V 16 w(START\(LOCAL)p 663 2391 V 15 w(COMM,)h(LOCAL)p 942 2391 V 16 w(LEADER,)f(PEER)p 1245 2391 V 17 w(COMM,)g(REMOTE)p 1549 2391 V 16 w(LEADER,)g(TAG,)393 2448 y(INTER)p 516 2448 V 17 w(REQUEST,)f(IERROR\))170 2504 y(INTEGER)h(LOCAL)p 484 2504 V 17 w(COMM,)g(LOCAL)p 764 2504 V 17 w(LEADER,)f(PEER)p 1067 2504 V 17 w(COMM,)h(REMOTE)p 1371 2504 V 17 w(LEADER,)g(TAG,)170 2561 y(INTER)p 293 2561 V 17 w(REQUEST,)g(IERROR)75 2647 y FC(This)d(call)g(starts)e (o\013)g(an)h(in)o(ter-comm)o(unicator)g(creation)g(op)q(eration,)h (returning)f(a)g(handle)h(for)f(the)75 2704 y(completion)h(of)e(the)h (op)q(eration)g(in)h Fv(inter)p 801 2704 14 2 v 17 w(request)p FC(;)i(it)d(is)g(collectiv)o(e)i(in)f Fv(lo)q(cal)p 1432 2704 V 16 w(comm)p FC(.)29 b(The)19 b(call)h(do)q(es)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 119 127 119 126 bop 75 -100 a Fs(5.5.)34 b(INTER-COMMUNICA)l(TION)1047 b FC(119)75 45 y(not)22 b(w)o(ait)g(for)g(the)h(remote)f(group)g(to)g (execute)h Fv(MPI)p 1046 45 14 2 v 16 w(INTERCOMM)p 1336 45 V 16 w(ST)l(ART\(\))p FC(.)42 b Fv(inter)p 1671 45 V 17 w(request)24 b FC(is)75 102 y(conceptually)19 b(similar)g(to)d (the)i(comm)o(unication)g(request)f(handle)i(used)e(b)o(y)h(non-blo)q (c)o(king)h(p)q(oin)o(t-to-)75 158 y(p)q(oin)o(t)13 b(routines.)20 b(The)12 b(handle)i(is)g(constructed)e(b)o(y)h(a)p 1002 158 V 29 w Fv(sta)o(rt)g FC(routine)g(and)g(destro)o(y)o(ed)f(b)o(y)g (the)h(matc)o(hing)p 75 214 V 89 214 a Fv(\014nish)20 b FC(routine;)h(these)f(handles)g(are)f(not)g(v)m(alid)h(for)f(an)o(y)g (other)f(use.)32 b(It)19 b(is)h(erroneous)f(to)f(call)j(this)75 271 y(routine)12 b(a)f(second)h(time)g(with)g(the)g(same)f Fv(b)o(ridge)p 908 271 V 16 w(comm)p FC(,)f Fv(remote)p 1196 271 V 16 w(leader)i FC(and)f Fv(tag)h FC(argumen)o(ts,)f(without) 75 327 y(an)k(in)o(terv)o(ening)h(call)h(to)d Fv(MPI)p 598 327 V 16 w(INTERCOMM)p 888 327 V 17 w(FINISH\(\))g FC(to)h(complete)g(the)h(p)q(ending)h(call.)166 386 y(Pro)q(cesses)h (should)h(pro)o(vide)g(iden)o(tical)h Fv(comm)p FC(,)c Fv(lo)q(cal)p 1106 386 V 16 w(leader)i FC(and)g Fv(tag)g FC(argumen)o(ts)g(within)h(eac)o(h)75 442 y(group.)75 548 y Fv(MPI)p 160 548 V 16 w(INTERCOMM)p 450 548 V 16 w(FINISH\(inter)p 711 548 V 17 w(request,)d(new)o(comm\))117 627 y Fq(INOUT)62 b Fv(inter)p 407 627 V 17 w(request)349 b Fq(handle)14 b(returned)i(b)o(y)d Fl(MPI)p 1342 627 13 2 v 15 w(INTERCOMM)p 1608 627 V 14 w(ST)m(ART\(\))117 705 y Fq(OUT)108 b Fv(new)o(comm)397 b Fq(new)15 b(in)o(ter-comm)o (unicator)c(\(handle\))75 832 y Fo(int)23 b(MPI)p 245 832 15 2 v 17 w(Intercomm)p 478 832 V 16 w(finish\(MPI)p 734 832 V 16 w(Intercomm)p 966 832 V 16 w(request)g(*inter)p 1317 832 V 16 w(request,)393 888 y(MPI)p 468 888 V 17 w(Comm)g(*newcomm\))75 976 y(MPI)p 150 976 V 17 w(INTERCOMM)p 383 976 V 16 w(FINISH\(INTER)p 687 976 V 15 w(REQUEST,)g(IERROR\))170 1033 y(INTEGER)g(INTER)p 484 1033 V 17 w(REQUEST,)g(NEWCOMM,)f(IERROR) 75 1121 y Fv(MPI)p 160 1121 14 2 v 16 w(INTERCOMM)p 450 1121 V 16 w(FINISH)d FC(completes)g(an)g(async)o(hronous)f(in)o (ter-comm)o(unicator)h(creation)g(op)q(er-)75 1178 y(ation.)27 b(This)18 b(routine)g(is)g(collectiv)o(e)h(in)g(the)e(goup)h(asso)q (ciated)g(with)f(comm)o(unicator)h Fv(lo)q(cal)p 1693 1178 V 16 w(comm)d FC(of)75 1234 y(the)h(corresp)q(onding)i(call)f(to)e Fv(MPI)p 673 1234 V 16 w(INTERCOMM)p 963 1234 V 17 w(ST)l(ART)p FC(.)i(It)f(w)o(aits)g(for)f(the)i(remote)e(group)h(to)g(call)75 1290 y Fv(MPI)p 160 1290 V 16 w(INTERCOMM)p 450 1290 V 16 w(ST)l(ART)h FC(but)e(do)q(es)g(not)g(w)o(ait)g(for)g(the)g (remote)g(group)f(to)h(call)75 1347 y Fv(MPI)p 160 1347 V 16 w(INTERCOMM)p 450 1347 V 16 w(FINISH)p FC(.)166 1481 y Fn(Implemen)o(tati)o(on)8 b(note:)33 b Fl(MPI)p 723 1481 13 2 v 14 w(INTERCOMM)p 988 1481 V 15 w(ST)m(ART)10 b Fq(initiates)g(async)o(hronous)h(sends)h(and)e(receiv)o(es)75 1531 y(for)i(an)h(exc)o(hange)g(of)g(information)d(b)q(et)o(w)o(een)k (leaders.)k Fl(MPI)p 1031 1531 V 15 w(INTERCOMM)p 1297 1531 V 14 w(FINISH)12 b Fq(completes)h(the)g(exc)o(hange)75 1581 y(of)g(information,)e(next)j(execute)i(a)e(broadcast)g(in)f(eac)o (h)i(lo)q(cal)e(group,)g(with)h(the)g(lo)q(cal)f(leader)h(at)g(the)h (ro)q(ot.)75 1769 y Fv(MPI)p 160 1769 14 2 v 16 w(INTERCOMM)p 450 1769 V 16 w(MERGE\(intercomm,)f(high,)i(newintracomm\))117 1848 y Fq(IN)155 b Fv(intercomm)384 b Fq(In)o(tercomm)o(unicator)12 b(\(handle\))117 1927 y(IN)155 b Fv(high)507 b Fq(\(logical\))117 2006 y(OUT)108 b Fv(newintracomm)308 b Fq(new)15 b(in)o(tra-comm)n (unicator)c(\(handle\))75 2132 y Fo(int)23 b(MPI)p 245 2132 15 2 v 17 w(Intercomm)p 478 2132 V 16 w(merge\(MPI)p 710 2132 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 2188 y(MPI)p 468 2188 V 17 w(Comm)g(*newintracomm\))75 2277 y(MPI)p 150 2277 V 17 w(INTERCOMM)p 383 2277 V 16 w(MERGE\(INTERCOMM,)e (HIGH,)j(NEWINTERCOMM,)e(IERROR\))170 2333 y(INTEGER)h(INTERCOMM,)g (NEWINTRACOMM,)f(IERROR)170 2390 y(LOGICAL)h(HIGH)75 2478 y FC(This)g(function)g(returns)g(an)f(in)o(tra-comm)o(unicator)g (for)g(the)h(union)g(of)g(the)f(t)o(w)o(o)f(groups)h(that)g(are)75 2534 y(asso)q(ciated)c(with)g Fv(intercomm)p FC(.)26 b(All)19 b(pro)q(cesses)f(should)h(pro)o(vide)f(the)g(same)f Fv(high)h FC(v)m(alue)h(within)g(eac)o(h)75 2591 y(of)d(the)g(t)o(w)o (o)e(groups.)22 b(If)16 b(pro)q(cesses)h(in)g(one)f(group)f(pro)o (vided)i(the)f(v)m(alue)h Fv(high)g(=)f(false)g FC(and)h(pro)q(cesses) 75 2647 y(in)h(the)g(other)f(group)g(pro)o(vided)i(the)e(v)m(alue)i Fv(high)f(=)g(true)g FC(then)g(the)f(union)i(orders)e(the)g(\\lo)o(w")g (group)75 2704 y(b)q(efore)g(the)g(\\high")g(group.)23 b(If)17 b(all)h(pro)q(cesses)f(pro)o(vided)g(the)g(same)g Fv(high)g FC(argumen)o(t)f(then)g(the)h(order)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 120 128 120 127 bop 75 -100 a FC(120)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y FC(of)e(the)g(union)i(is)f(arbitrary)l(.)k(This)c(call)h(is)f(blo) q(c)o(king)g(and)g(collectiv)o(e)h(within)g(the)e(union)h(of)f(the)h (groups.)75 171 y Fp(5.5.3)49 b(Name)16 b(Service)75 258 y FC(MPI)e(pro)o(vides)h(a)g(name)f(service)h(to)f(simplify)j (construction)d(of)g(in)o(ter-comm)o(unicators.)20 b(This)15 b(service)75 315 y(allo)o(ws)f(a)g(lo)q(cal)h(pro)q(cess)f(group)g(to)f (create)h(an)g(in)o(ter-comm)o(unicator)g(when)g(the)g(only)h(a)o(v)m (ailable)h(infor-)75 371 y(mation)10 b(ab)q(out)h(the)f(remote)g(group) h(is)g(a)f(user-de\014ned)i(c)o(haracter)e(string.)18 b(A)11 b(lo)q(osely)h(sync)o(hronous)e(v)o(er-)75 428 y(sion)h(is)f(pro)o(vided)h(b)o(y)f(routines)h Fv(MPI)p 699 428 14 2 v 16 w(INTERCOMM)p 989 428 V 16 w(NAME)p 1134 428 V 17 w(ST)l(ART)g FC(and)g Fv(MPI)p 1470 428 V 15 w(INTERCOMM)p 1759 428 V 17 w(FINISH)p FC(.)75 484 y(Note)22 b(that)g(for)g(rendezv)o(ous)h(to)f(o)q(ccur,)j(all)e(in)o(v) o(o)q(cations)h(of)e Fv(MPI)p 1283 484 V 16 w(INTERCOMM)p 1573 484 V 16 w(NAME)p 1718 484 V 17 w(ST)l(ART)75 541 y FC(clearly)16 b(m)o(ust)f(sp)q(ecify)h(the)g(same)f(v)m(alue)h(for)f (the)g Fv(name)g FC(argumen)o(t.)75 645 y Fv(MPI)p 160 645 V 16 w(INTERCOMM)p 450 645 V 16 w(NAME)p 595 645 V 17 w(ST)l(ART\(comm,)f(name,)g(inter)p 1131 645 V 17 w(request\))117 723 y Fq(IN)155 b Fv(comm)470 b Fq(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e(\(handle\))117 800 y(IN)155 b Fv(name)485 b Fq(c)o(haracter)15 b(string)117 876 y(OUT)108 b Fv(inter)p 407 876 V 17 w(request)349 b Fq(handle)14 b(for)g(MPI)p 1186 876 13 2 v 15 w(INTER)o(COMM)p 1473 876 V 15 w(FINISH\(\))75 1001 y Fo(int)23 b(MPI)p 245 1001 15 2 v 17 w(Intercomm)p 478 1001 V 16 w(name)p 590 1001 V 17 w(start\(MPI)p 823 1001 V 16 w(Comm)g(comm,)g(char)h(*name,) 393 1058 y(MPI)p 468 1058 V 17 w(Intercomm)p 701 1058 V 16 w(request)f(*inter)p 1052 1058 V 16 w(request\))75 1145 y(MPI)p 150 1145 V 17 w(INTERCOMM)p 383 1145 V 16 w(NAME)p 495 1145 V 16 w(START\(COMM,)g(NAME,)g(NEW)p 1013 1145 V 17 w(COMM,)g(INTER)p 1293 1145 V 16 w(REQUEST,)g(IERROR\)) 170 1201 y(CHARACTER)g(*\(*\))g(NAME)170 1258 y(INTEGER)g(COMM,)h(NEW)p 580 1258 V 16 w(COMM,)f(INTER)p 859 1258 V 17 w(REQUEST,)g(IERROR)75 1345 y FC(This)d(is)h(the)f(name-serv)o(ed)g(equiv)m(alen)o(t)h(of)f Fv(MPI)p 944 1345 14 2 v 16 w(INTERCOMM)p 1234 1345 V 16 w(ST)l(ART)p FC(.)h(The)f(op)q(eration)g(is)g(com-)75 1401 y(pleted)c(b)o(y)f(a)g(call)i(to)d Fv(MPI)p 536 1401 V 16 w(INTERCOMM)p 826 1401 V 17 w(FINISH)p FC(.)75 1549 y Ft(5.6)59 b(Caching)75 1652 y FC(MPI)22 b(pro)o(vides)h(a)e (\\cac)o(hing")h(facilit)o(y)i(that)d(allo)o(ws)h(an)g(application)i (to)d(attac)o(h)g(arbitrary)h(pieces)75 1708 y(of)17 b(information,)h(called)h Fx(attributes)p FC(,)g(to)e(comm)o (unicators.)27 b(More)17 b(precisely)l(,)j(the)e(cac)o(hing)g(facilit)o (y)75 1765 y(allo)o(ws)d(a)g(p)q(ortable)h(library)g(to)e(do)i(the)f (follo)o(wing:)143 1861 y FA(\017)23 b FC(pass)15 b(information)g(b)q (et)o(w)o(een)g(calls)i(b)o(y)e(asso)q(ciating)g(it)h(with)f(an)h(MPI)f (comm)o(unicator,)143 1958 y FA(\017)23 b FC(quic)o(kly)16 b(retriev)o(e)g(that)e(information,)h(and)143 2055 y FA(\017)23 b FC(b)q(e)18 b(guaran)o(teed)e(that)h(out-of-date)g (information)g(is)h(nev)o(er)g(retriev)o(ed,)f(ev)o(en)h(if)g(the)f (comm)o(uni-)189 2111 y(cator)d(is)i(freed)f(and)h(its)f(handle)h (subsequen)o(tly)h(reused)f(b)o(y)f(MPI.)75 2208 y(These)j (capabilities,)i(in)f(some)e(form,)g(are)h(required)g(b)o(y)g(built-in) i(MPI)e(routines)g(suc)o(h)g(as)f(collectiv)o(e)75 2265 y(comm)o(unication)g(and)g(virtual)g(top)q(ology)l(.)23 b(De\014ning)17 b(an)g(in)o(terface)f(to)g(these)h(capabilities)i(as)d (part)f(of)75 2321 y(the)f(MPI)f(standard)g(is)h(v)m(aluable)h(b)q (ecause)g(it)f(p)q(ermits)g(routines)g(lik)o(e)g(collectiv)o(e)i(comm)o (unication)e(and)75 2378 y(virtual)h(top)q(ology)g(to)f(b)q(e)h (implemen)o(ted)h(as)f(p)q(ortable)g(co)q(de,)g(and)g(also)f(b)q (ecause)i(it)f(mak)o(es)f(MPI)g(more)75 2434 y(extensible)j(b)o(y)e (allo)o(wing)h(user-written)g(routines)f(to)g(use)h(standard)e(MPI)h (calling)i(sequences.)75 2560 y Fp(5.6.1)49 b(F)o(unctionalit)o(y)75 2647 y FC(A)o(ttributes)21 b(are)f(lo)q(cal)i(to)e(the)h(pro)q(cess)g (and)g(sp)q(eci\014c)i(to)d(the)h(comm)o(unicator)g(to)f(whic)o(h)h (they)g(are)75 2704 y(attac)o(hed.)d(They)13 b(are)e(not)h(propagated)f (b)o(y)h(MPI)g(from)g(one)g(comm)o(unicator)g(to)f(another)h(except)g (when)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 121 129 121 128 bop 75 -100 a Fs(5.6.)34 b(CA)o(CHING)1407 b FC(121)75 45 y(the)11 b(comm)o(unicator)f(is)h(duplicated)i(using)e Fv(MPI)p 899 45 14 2 v 16 w(COMM)p 1057 45 V 16 w(DUP)g FC(\(and)g(ev)o(en)g(then)g(the)f(application)j(m)o(ust)75 102 y(giv)o(e)j(sp)q(eci\014c)i(p)q(ermission)f(for)e(the)h(attribute)g (to)f(b)q(e)i(copied\).)22 b(A)o(ttributes)16 b(are)g(scalar)f(v)m (alues,)i(equal)75 158 y(in)f(size)g(to,)e(or)h(larger)g(than)g(a)g (C-language)g(p)q(oin)o(ter.)166 219 y(The)10 b(cac)o(hing)h(in)o (terface)g(de\014ned)g(here)g(represen)o(ts)f(that)f(attributes)h(are)g (stored)g(b)o(y)g(MPI)g(opaquely)75 276 y(within)16 b(a)f(comm)o (unicator.)20 b(Accessor)15 b(functions)h(include)i(the)d(follo)o (wing:)143 390 y FA(\017)23 b FC(obtain)16 b(a)f(k)o(ey)h(v)m(alue)h (\(used)f(to)g(iden)o(tify)h(an)f(attribute\);)f(the)h(user)g(sp)q (eci\014es)i(\\callbac)o(k")e(func-)189 446 y(tions)g(b)o(y)h(whic)o(h) h(MPI)e(informs)h(the)g(application)h(when)f(the)g(comm)o(unicator)f (is)h(destro)o(y)o(ed)f(or)189 502 y(copied.)143 616 y FA(\017)23 b FC(store)14 b(and)i(retriev)o(e)f(the)g(v)m(alue)i(of)d (an)i(attribute;)75 730 y(MPI)f(pro)o(vides)h(the)f(follo)o(wing)h (services)g(related)g(to)e(cac)o(hing.)21 b(They)15 b(are)g(all)h(lo)q (cal.)75 839 y Fv(MPI)p 160 839 V 16 w(A)l(TTR)p 293 839 V 17 w(CREA)l(TE)p 479 839 V 17 w(KEY\(cop)o(y)p 689 839 V 17 w(fn,)f(delete)p 881 839 V 17 w(fn,)h(k)o(eyval,)e(extra)p 1202 839 V 16 w(state\))117 921 y Fq(IN)155 b Fv(cop)o(y)p 407 921 V 17 w(fn)447 b Fq(Cop)o(y)14 b(callbac)o(k)f(function)g(for)h Fl(k)o(eyval)117 1006 y Fq(IN)155 b Fv(delete)p 431 1006 V 18 w(fn)422 b Fq(Delete)15 b(callbac)o(k)e(function)g(for)h Fl(k)o(eyval)117 1091 y Fq(OUT)108 b Fv(k)o(eyval)471 b Fq(k)o(ey)14 b(v)n(alue)f(for)h(future)g(access)i(\(in)o(teger\))117 1176 y(IN)155 b Fv(extra)p 416 1176 V 17 w(state)384 b Fq(Extra)14 b(state)h(for)f(callbac)o(k)f(functions)75 1305 y Fo(int)23 b(MPI)p 245 1305 15 2 v 17 w(Attr)p 358 1305 V 17 w(create)p 519 1305 V 16 w(key\(int)g(copy)p 822 1305 V 17 w(fn\(\),)g(int)g(delete)p 1221 1305 V 17 w(fn\(\),)g(int)g(keyval,)393 1362 y(void*)g(extra)p 659 1362 V 17 w(state\))75 1453 y(MPI)p 150 1453 V 17 w(ATTR)p 263 1453 V 16 w(CREATE)p 423 1453 V 17 w(KEY\(COPY)p 632 1453 V 16 w(FN,)g(DELETE)p 887 1453 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1310 1453 V 17 w(STATE,)g(IERROR\))170 1510 y(EXTERNAL)g(COPY)p 484 1510 V 17 w(FN,)g(DELETE)p 740 1510 V 17 w(FN)170 1566 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 1566 V 17 w(STATE,)g(IERROR)166 1658 y FC(Generates)13 b(a)g(new)h(attribute)g(k)o(ey)l(.)19 b(Keys)14 b(are)g(lo)q(cally)h(unique)g(in)g(pro)q(cess,)f(and)f (opaque)h(to)f(user,)75 1714 y(though)i(they)h(are)f(explicitly)k (stored)c(in)h(in)o(tegers.)21 b(Once)c(allo)q(cated,)f(the)f(k)o(ey)h (v)m(alue)h(can)e(b)q(e)h(used)h(to)75 1770 y(asso)q(ciate)e (attributes)g(and)h(access)f(them)g(on)g(an)o(y)g(lo)q(cally)i (de\014ned)g(comm)o(unicator.)166 1832 y(The)h Fv(cop)o(y)p 351 1832 14 2 v 17 w(fn)g FC(function)h(is)g(in)o(v)o(ok)o(ed)f(when)h (a)f(comm)o(unicator)g(is)h(duplicated.)30 b(A)19 b(C)f(protot)o(yp)q (e)75 1888 y(for)d(suc)o(h)g(a)g(function)h(is)166 1950 y Fv(int)g(cop)o(y)p 321 1950 V 16 w(fn\(oldcomm,)d(new)o(comm,)h(k)o (eyval,)g(extra)p 1047 1950 V 17 w(state\))166 2011 y FC(This)22 b(function)h(do)q(es)g(whatev)o(er)e(it)h(needs)h(to)e(do)h (with)g(the)h(new)f(comm)o(unicator)f Fv(new)o(comm)75 2068 y FC(based)16 b(on)f(what)f(attribute)h(information)h(w)o(as)e(in) i(the)f(old)h(comm)o(unicator)f Fv(oldcomm)p FC(;)e(it)j(returns)f(1)f (if)75 2124 y(successful,)h(0)e(if)h(not.)19 b(A)14 b(v)m(alid)h(cop)o (y)f(function)g(is)g(one)g(that)f(completely)i(duplicates)g(the)f (information)75 2181 y(b)o(y)20 b(making)g(a)f(full)i(duplicate)g(cop)o (y)f(of)f(the)h(data)f(structures)h(implied)i(b)o(y)d(an)h(attribute;)h (another)75 2237 y(migh)o(t)f(just)g(mak)o(e)g(another)g(reference)h (to)e(that)h(data)g(structure,)g(while)i(using)f(a)f(reference-coun)o (t)75 2293 y(mec)o(hanism.)37 b(Other)21 b(t)o(yp)q(es)g(of)f (attributes)h(migh)o(t)g(not)f(cop)o(y)h(at)f(all)h(\(they)g(migh)o(t)g (b)q(e)g(sp)q(eci\014c)i(to)75 2350 y Fv(oldcomm)15 b FC(only\).)24 b Fv(MPI)p 488 2350 V 16 w(COMM)p 646 2350 V 16 w(DUP)17 b FC(is)g(the)g(only)g(function)g(that)f(calls)i(the)e Fv(cop)o(y)p 1547 2350 V 17 w(fn)p FC(.)24 b Fv(cop)o(y)p 1724 2350 V 17 w(fn)17 b FC(ma)o(y)75 2406 y(b)q(e)f(sp)q(eci\014ed)h (as)e(n)o(ull,)h(in)g(whic)o(h)g(case)g(no)f(cop)o(y)g(callbac)o(k)h(o) q(ccurs)g(for)e Fv(k)o(eyval)p FC(.)166 2468 y(Analogous)h(to)g Fv(cop)o(y)p 532 2468 V 16 w(fn)h FC(is)f(a)g(callbac)o(k)i(deletion)f (function,)g(de\014ned)g(as)f(follo)o(ws:)166 2529 y Fv(delete)p 279 2529 V 18 w(fn\(comm,)e(k)o(eyval,)h(extra)p 733 2529 V 16 w(state\))166 2591 y FC(This)e(function)g(is)g(called)i (b)o(y)d Fv(MPI)p 752 2591 V 16 w(COMM)p 910 2591 V 16 w(FREE)h FC(and)g Fv(MPI)p 1214 2591 V 16 w(A)l(TTR)p 1347 2591 V 17 w(DELETE)f FC(to)g(do)h(whatev)o(er)e(is)75 2647 y(needed)15 b(to)f(remo)o(v)o(e)f(an)h(attribute.)19 b(It)14 b(ma)o(y)f(b)q(e)i(sp)q(eci\014ed)h(as)d(n)o(ull,)j(in)e(whic)o (h)h(case)f(no)g(delete)h(callbac)o(k)75 2704 y(o)q(ccurs)h(for)e Fv(k)o(eyval)p FC(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 122 130 122 129 bop 75 -100 a FC(122)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(A)l(TTR)p 293 45 V 17 w(FREE)p 419 45 V 17 w(KEY\(k)o(eyval\))117 122 y Fq(IN)155 b Fv(k)o(eyval)471 b Fq(F)m(rees)15 b(the)g(in)o(teger)f(k)o(ey)g(v)n (alue)f(\(in)o(teger\))75 247 y Fo(int)23 b(MPI)p 245 247 15 2 v 17 w(Attr)p 358 247 V 17 w(free)p 471 247 V 16 w(key\(int)g(keyval\))75 333 y(MPI)p 150 333 V 17 w(ATTR)p 263 333 V 16 w(FREE)p 375 333 V 17 w(KEY\(KEYVAL,)f(IERROR\)) 170 390 y(INTEGER)h(KEYVAL,)g(IERROR)166 476 y FC(F)l(rees)13 b(an)g(extan)o(t)f(attribute)h(k)o(ey)l(.)19 b(It)14 b(is)f(not)g(erroneous)g(to)f(free)h(an)g(attribute)g(k)o(ey)g(that)g (is)g(in)h(use,)75 533 y(b)q(ecause)h(the)e(actual)h(free)f(do)q(es)h (not)f(transpire)h(un)o(til)h(after)e(all)h(references)g(\(in)g(other)f (comm)o(unicators)75 589 y(on)20 b(the)g(pro)q(cess\))f(to)h(the)g(k)o (ey)f(ha)o(v)o(e)h(b)q(een)h(freed.)34 b(These)20 b(references)g(need)h (to)e(b)q(e)i(explictly)h(freed)75 645 y(b)o(y)d(the)h(program,)f (either)h(via)f(calls)i(to)e Fv(MPI)p 887 645 14 2 v 15 w(A)l(TTR)p 1019 645 V 18 w(DELETE)g FC(that)f(free)i(one)f (attribute)h(instance,)75 702 y(or)c(b)o(y)g(calls)h(to)e Fv(MPI)p 439 702 V 16 w(COMM)p 597 702 V 17 w(FREE)h FC(that)g(free)g(all)h(attribute)f(instances)h(asso)q(ciated)f(with)g (the)h(freed)75 758 y(comm)o(unicator.)75 862 y Fv(MPI)p 160 862 V 16 w(PUT)p 267 862 V 16 w(A)l(TTR\(comm,)d(k)o(eyval,)g (attribute)p 868 862 V 18 w(val\))117 939 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)7 b(to)i(whic)o(h)g(attribute)h (will)e(b)q(e)h(attac)o(hed)h(\(han-)905 996 y(dle\))117 1069 y(IN)155 b Fv(k)o(eyval)471 b Fq(k)o(ey)14 b(v)n(alue,)f(as)h (returned)h(b)o(y)905 1126 y Fl(MPI)p 982 1126 13 2 v 15 w(A)m(TTR)p 1105 1126 V 14 w(CREA)m(TE)p 1276 1126 V 13 w(KEY)e Fq(\(in)o(teger\))117 1199 y(IN)155 b Fv(attribute)p 484 1199 14 2 v 18 w(val)352 b Fq(attribute)15 b(v)n(alue)75 1324 y Fo(int)23 b(MPI)p 245 1324 15 2 v 17 w(Put)p 334 1324 V 17 w(attr\(MPI)p 543 1324 V 16 w(Comm)g(comm,)h(int)f(keyval,)g (void*)g(attribute)p 1467 1324 V 16 w(val\))75 1410 y(MPI)p 150 1410 V 17 w(PUT)p 239 1410 V 17 w(ATTR\(COMM,)f(KEYVAL,)h (ATTRIBUTE)p 925 1410 V 16 w(VAL,)g(IERROR\))170 1467 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 1467 V 16 w(VAL,)i(IERROR)166 1553 y FC(This)14 b(function)g(stores)f(the)g (stipulated)i(attribute)e(v)m(alue)i Fv(attribute)p 1331 1553 14 2 v 18 w(val)e FC(for)g(subsequen)o(t)h(retriev)m(al)75 1610 y(b)o(y)c Fv(MPI)p 218 1610 V 16 w(GET)p 322 1610 V 17 w(A)l(TTR)p FC(.)g(If)h(the)f(v)m(alue)h(is)g(already)f(presen)o (t,)h(then)f(the)h(outcome)e(is)i(as)f(if)h Fv(MPI)p 1627 1610 V 15 w(DELETE)p 1812 1610 V 16 w(A)l(TTR)75 1666 y FC(w)o(as)17 b(\014rst)h(called)i(to)e(delete)h(the)f(previous)h (v)m(alue)h(\(and)e(the)g(callbac)o(k)i(function)f Fv(delete)p 1635 1666 V 17 w(fn)g FC(w)o(as)e(exe-)75 1722 y(cuted\),)d(and)h(a)e (new)i(v)m(alue)g(w)o(as)f(next)g(stored.)19 b(The)14 b(call)i(is)e(erroneous)g(if)h(there)f(is)h(no)f(k)o(ey)g(with)h(v)m (alue)75 1779 y Fv(k)o(eyval)p FC(.)75 1883 y Fv(MPI)p 160 1883 V 16 w(GET)p 264 1883 V 17 w(A)l(TTR\(comm,)e(k)o(eyval,)h (attribute)p 865 1883 V 18 w(val,)h(found\))117 1960 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)10 b(to)i(whic)o(h)g (attribute)h(is)f(attac)o(hed)h(\(handle\))117 2033 y(IN)155 b Fv(k)o(eyval)471 b Fq(k)o(ey)14 b(v)n(alue)f(\(in)o(teger\))117 2107 y(OUT)108 b Fv(attribute)p 484 2107 V 18 w(val)352 b Fq(attribute)15 b(v)n(alue,)d(unless)j Fl(found)g Fq(=)f(false)117 2180 y(OUT)108 b Fv(found)481 b Fk(true)14 b Fq(if)h(an)f(attribute)i (v)n(alue)e(w)o(as)h(extracted;)i Fk(false)d Fq(if)g(no)905 2237 y(attribute)h(is)e(asso)q(ciated)i(with)f(the)g(k)o(ey)75 2361 y Fo(int)23 b(MPI)p 245 2361 15 2 v 17 w(Get)p 334 2361 V 17 w(attr\(MPI)p 543 2361 V 16 w(Comm)g(comm,)h(int)f(keyval,)g (void*)g(attribute)p 1467 2361 V 16 w(val,)g(int)h(*found\))75 2448 y(MPI)p 150 2448 V 17 w(GET)p 239 2448 V 17 w(ATTR\(COMM,)e (KEYVAL,)h(ATTRIBUTE)p 925 2448 V 16 w(VAL,)g(FOUND,)g(IERROR\))170 2504 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 2504 V 16 w(VAL,)i(IERROR)170 2561 y(LOGICAL)f(FOUND)166 2647 y FC(Retriev)o(es)c(attribute)g(v)m(alue)h(b)o(y)f(k)o(ey)l(.)30 b(The)19 b(call)h(is)f(erroneous)g(if)g(there)g(is)g(no)g(k)o(ey)g (with)g(v)m(alue)75 2704 y Fv(k)o(eyval)p FC(.)30 b(On)19 b(the)g(other)f(hand,)h(the)g(call)h(is)f(correct)f(if)h(the)g(k)o(ey)f (v)m(alue)i(exists,)g(but)e(no)h(attribute)f(is)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 123 131 123 130 bop 75 -100 a Fs(5.6.)29 b(CA)o(CHING)1412 b FC(123)75 45 y(attac)o(hed)15 b(on)g Fo(comm)f FC(for)h(that)f(k)o(ey;) h(in)h(suc)o(h)g(case,)e(the)i(call)g(returns)f Fo(found)23 b(=)h(false)p FC(.)75 149 y Fv(MPI)p 160 149 14 2 v 16 w(DELETE)p 346 149 V 16 w(A)l(TTR\(comm,)13 b(k)o(eyval\))117 226 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed)h(\(handle\))117 300 y(IN)155 b Fv(k)o(eyval)471 b Fq(The)15 b(k)o(ey)e(v)n(alue)h(of)f (the)i(deleted)f(attribute)h(\(in)o(teger\))75 424 y Fo(int)23 b(MPI)p 245 424 15 2 v 17 w(Delete)p 406 424 V 17 w(Attr\(MPI)p 615 424 V 16 w(Comm)g(comm,)g(int)h(keyval\))75 511 y(MPI)p 150 511 V 17 w(DELETE)p 311 511 V 16 w(ATTR\(COMM,)f (KEYVAL,)g(IERROR\))170 567 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)166 653 y FC(Delete)14 b(attribute)g(from)f(cac)o(he)h(b)o(y)f(k)o(ey)l(.) 20 b(This)14 b(function)g(in)o(v)o(ok)o(es)g(the)g(attribute)f(delete)i (function)75 710 y Fv(delete)p 188 710 14 2 v 18 w(fn)g FC(sp)q(eci\014ed)j(when)d(the)h Fv(k)o(eyval)e FC(w)o(as)h(created.) 166 766 y(Whenev)o(er)h(a)g(comm)o(unicator)f(is)h(replicated)i(using)e (the)g(function)h Fv(MPI)p 1435 766 V 16 w(COMM)p 1593 766 V 16 w(DUP)p FC(,)f(all)h(call-)75 823 y(bac)o(k)h(cop)o(y)f (functions)i(for)e(attributes)g(that)g(are)h(curren)o(tly)g(set)f(are)h (in)o(v)o(ok)o(ed)g(\(in)g(arbitrary)f(order\).)75 879 y(Whenev)o(er)23 b(a)f(comm)o(unicator)g(is)g(deleted)i(using)f(the)g (function)g Fv(MPI)p 1343 879 V 15 w(COMM)p 1500 879 V 17 w(FREE)g FC(all)g(callbac)o(k)75 936 y(delete)16 b(functions)g(for)f(attributes)g(that)f(are)h(curren)o(tly)h(set)f(are) g(in)o(v)o(ok)o(ed.)75 1054 y Fv(Example)75 1140 y FC(This)j(example)g (sho)o(ws)e(ho)o(w)h(to)f(write)i(a)e(collectiv)o(e)j(comm)o(unication) f(op)q(eration)f(that)g(uses)g(cac)o(hing)75 1197 y(to)d(b)q(e)h(more)f (e\016cien)o(t)h(after)f(the)h(\014rst)f(call.)21 b(The)14 b(co)q(ding)i(st)o(yle)f(assumes)f(that)g(MPI)g(function)i(results)75 1253 y(return)f(only)h(error)e(statuses.)147 1349 y Fo(static)23 b(int)g(gop_key_assigned)f(=)h(0;)48 b(/*)24 b(0)f(only)h(on)f(first)g (entry)h(*/)147 1405 y(static)f(int)g(gop_key;)357 b(/*)24 b(key)f(for)g(this)h(module's)f(stuff)g(*/)147 1518 y(typedef)f(struct) 147 1574 y({)218 1631 y(int)i(ref_count;)237 b(/*)23 b(reference)g(count)g(*/)218 1687 y(/*)h(other)f(stuff,)g(whatever)g (else)g(we)h(want)f(*/)147 1744 y(})g(gop_stuff_type;)147 1857 y(Efficient_Collective_Op)d(\(comm,)j(...\))147 1913 y(MPI_Comm)f(comm;)147 1970 y({)194 2026 y(gop_stuff_type)g (*gop_stuff;)194 2083 y(MPI_Group)166 b(group;)194 2139 y(int)310 b(foundflag;)194 2252 y(MPI_Comm_group\(comm,)21 b(&group\);)194 2365 y(if)j(\(!gop_key_assigned\))117 b(/*)23 b(get)h(a)f(key)h(on)f(first)h(call)f(ever)g(*/)194 2421 y({)242 2478 y(gop_key_assigned)f(=)h(1;)242 2534 y(if)h(\()f(!)h(MPI_Attr_get_key\()e(gop_stuff_copier,)839 2591 y(gop_stuff_destructor,)839 2647 y(&gop_key,)g(\(void)h(*\)0\)\);) 242 2704 y(/*)h(get)f(the)h(key)f(while)g(assigning)g(its)g(copy)h(and) f(delete)g(callback)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 124 132 124 131 bop 75 -100 a FC(124)123 b Fs(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)314 45 y Fo(behavior.)22 b(*/)242 158 y(MPI_Abort)h(\("Insufficient)f(keys) h(available"\);)194 214 y(})194 327 y(MPI_Attr_get_value)f(\(comm,)h (gop_key,)g(&gop_stuff,)f(&foundflag\);)194 384 y(if)i(\(foundflag\)) 194 440 y({)g(/*)g(This)f(module)g(has)g(executed)g(in)h(this)f(group)g (before.)314 497 y(We)g(will)h(use)f(the)g(cached)g(information)g(*/) 194 553 y(})194 610 y(else)194 666 y({)h(/*)g(This)f(is)h(a)f(group)g (that)h(we)f(have)h(not)f(yet)h(cached)f(anything)f(in.)314 723 y(We)h(will)h(now)f(do)h(so.)242 779 y(*/)242 892 y(/*)g(First,)f(allocate)f(storage)h(for)h(the)f(stuff)g(we)h(want,)314 948 y(and)f(initialize)g(the)g(reference)g(count)g(*/)242 1061 y(gop_stuff)g(=)g(\(gop_stuff_type)f(*\))i(malloc)f (\(sizeof\(gop_stuff_type\)\);)242 1118 y(if)h(\(gop_stuff)e(==)i (NULL\))f({)h(/*)f(abort)g(on)h(out-of-memory)e(error)h(*/)h(})242 1231 y(gop_stuff)f(->)g(ref_count)g(=)h(1;)242 1344 y(/*)g(Second,)f (fill)g(in)g(*gop_stuff)g(with)g(whatever)g(we)h(want.)314 1400 y(This)f(part)g(isn't)g(shown)h(here)f(*/)242 1513 y(/*)h(Third,)f(store)g(gop_stuff)f(as)i(the)g(attribute)e(value)h(*/) 242 1569 y(MPI_Attr_put_value)e(\()j(comm,)f(gop_key,)g(gop_stuff\);) 194 1626 y(})194 1682 y(/*)h(Then,)f(in)h(any)f(case,)g(use)h(contents) f(of)g(*gop_stuff)266 1739 y(to)g(do)h(the)g(global)f(op)g(...)h(*/)147 1795 y(})147 1908 y(/*)f(The)h(following)e(routine)h(is)h(called)f(by)g (MPI)h(when)f(a)h(group)f(is)h(freed)f(*/)147 2021 y (gop_stuff_destructor)e(\(comm,)i(keyval,)g(gop_stuff,)f(extra\))147 2077 y(MPI_Comm)g(comm;)147 2134 y(int)h(keyval;)147 2190 y(gop_stuff_type)f(*gop_stuff;)147 2247 y(void)h(*extra;)147 2303 y({)194 2360 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h (--)f(programming)g(error)g(*/)g(})194 2473 y(/*)h(The)f(group's)g (being)g(freed)h(removes)f(one)g(reference)g(to)g(gop_stuff)g(*/)194 2529 y(gop_stuff)g(->)h(ref_count)e(-=)i(1;)194 2642 y(/*)g(If)f(no)h(references)f(remain,)f(then)i(free)f(the)h(storage)e (*/)194 2698 y(if)i(\(gop_stuff)e(->)i(ref_count)f(==)g(0\))h({)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 125 133 125 132 bop 75 -100 a Fs(5.7.)29 b(F)o(ORMALIZING)11 b(THE)g(LOOSEL)l(Y)i(SYNCHR)o(ONOUS)g(MODEL)e(\(USA)o(GE,)f(SAFETY\))p FC(125)242 45 y Fo(free\(\(void)23 b(*\)gop_stuff\);)194 102 y(})147 158 y(})147 271 y(/*)g(The)h(following)e(routine)h(is)h (called)f(by)g(MPI)h(when)f(a)h(group)f(is)h(copied)f(*/)147 327 y(gop_stuff_copier)e(\(comm,)i(keyval,)g(gop_stuff,)g(extra\))147 384 y(MPI_Comm)f(comm;)147 440 y(int)h(keyval;)147 497 y(gop_stuff_type)f(*gop_stuff;)147 553 y(void)h(*extra;)147 610 y({)194 666 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h(--) f(programming)g(error)g(*/)g(})194 779 y(/*)h(The)f(new)h(group)f(adds) g(one)h(reference)e(to)i(this)f(gop_stuff)g(*/)194 835 y(gop_stuff)g(->)h(ref_count)e(+=)i(1;)147 892 y(})75 1046 y Ft(5.7)59 b(F)n(o)n(rmalizing)21 b(the)e(Lo)r(osely)g (Synchronous)g(Mo)r(del)g(\(Usage,)g(Safet)n(y\))75 1153 y Fp(5.7.1)49 b(Basic)17 b(Statements)75 1243 y FC(When)e(a)g(caller)h (passes)e(a)h(comm)o(unicator)f(\(that)g(con)o(tains)h(a)g(con)o(text)f (and)h(group\))f(to)g(a)h(callee,)h(that)75 1299 y(comm)o(unicator)f(m) o(ust)h(b)q(e)g(free)g(of)f(side)i(e\013ects)e(throughout)g(execution)i (of)e(the)h(subprogram)f(\(quies-)75 1355 y(cen)o(t\).)26 b(This)18 b(pro)o(vides)g(one)f(mo)q(del)h(in)g(whic)o(h)h(libraries)g (can)e(b)q(e)h(written,)f(and)h(w)o(ork)e(\\safely)l(.")27 b(F)l(or)75 1412 y(libraries)13 b(so)f(designated,)h(the)f(callee)h (has)f(p)q(ermission)h(to)e(do)h(whatev)o(er)f(comm)o(unication)i(it)f (lik)o(es)h(with)75 1468 y(the)19 b(comm)o(unicator,)g(and)g(under)h (the)f(ab)q(o)o(v)o(e)g(guaran)o(tee)f(kno)o(ws)g(that)h(no)g(other)f (comm)o(unications)75 1525 y(will)d(in)o(terfere.)k(Since)c(w)o(e)e(p)q (ermit)g(go)q(o)q(d)g(implemen)o(tations)i(to)d(create)h(new)g(comm)o (unicators)g(without)75 1581 y(sync)o(hronization)h(\(suc)o(h)g(as)f(b) o(y)g(preallo)q(cated)i(con)o(texts)e(on)g(comm)o(unicators\),)g(this)g (do)q(es)h(not)f(imp)q(ose)75 1638 y(a)i(signi\014can)o(t)h(o)o(v)o (erhead.)166 1696 y(This)k(form)f(of)g(safet)o(y)f(is)i(analogous)f(to) g(other)g(common)g(computer-science)j(usages,)e(suc)o(h)f(as)75 1753 y(passing)c(a)f(descriptor)h(of)f(an)h(arra)o(y)e(to)h(a)g (library)i(routine.)k(The)15 b(library)g(routine)g(has)g(ev)o(ery)f (righ)o(t)h(to)75 1809 y(exp)q(ect)h(suc)o(h)f(a)g(descriptor)h(to)f(b) q(e)g(v)m(alid)i(and)f(mo)q(di\014able.)75 1942 y Fp(5.7.2)49 b(Mo)q(dels)17 b(of)g(Execution)75 2031 y FC(W)l(e)i(sa)o(y)f(that)g(a) g(parallel)j(pro)q(cedure)e(is)g Fr(active)g FC(in)h(a)e(pro)q(cess)h (if)g(the)g(pro)q(cess)g(b)q(elongs)h(to)e(a)g(group)75 2088 y(that)e(ma)o(y)h(collectiv)o(ely)i(execute)f(the)f(pro)q(cedure,) h(and)f(some)f(mem)o(b)q(er)h(of)g(that)f(group)h(is)g(curren)o(tly)75 2144 y(executing)c(the)f(pro)q(cedure)h(co)q(de.)19 b(If)13 b(a)e(parallel)j(pro)q(cedure)f(is)f(activ)o(e)g(in)h(a)f(pro)q(cess,)g (then)g(this)h(pro)q(cess)75 2201 y(ma)o(y)e(b)q(e)i(receiving)g (messages)e(p)q(ertaining)i(to)f(this)g(pro)q(cedure,)h(ev)o(en)f(if)g (it)g(do)q(es)h(not)e(curren)o(tly)h(execute)75 2257 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 2388 y Fv(Non-reentrant)i(pa)o(rallel)d(p)o(ro)q(cedures)75 2478 y FC(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o(y)g (p)q(oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(o)q(cation)g(of) f(a)g(parallel)75 2534 y(pro)q(cedure)14 b(can)f(b)q(e)g(activ)o(e)g (at)f(an)o(y)h(pro)q(cess.)19 b(That)12 b(is,)i(concurren)o(t)e(in)o(v) o(o)q(cations)i(of)e(the)h(same)f(parallel)75 2591 y(pro)q(cedure)k(ma) o(y)e(o)q(ccur)h(only)g(within)h(disjoin)o(t)f(groups)f(of)h(pro)q (cesses.)20 b(F)l(or)14 b(example,)h(all)h(in)o(v)o(o)q(cations)75 2647 y(of)f(parallel)i(pro)q(cedures)f(in)o(v)o(olv)o(e)g(all)h(pro)q (cesses,)e(pro)q(cesses)h(are)f(single-threaded,)i(and)e(there)h(are)f (no)75 2704 y(recursiv)o(e)h(in)o(v)o(o)q(cations.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 126 134 126 133 bop 75 -100 a FC(126)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)166 45 y FC(In)21 b(suc)o(h)f(a)g(case,)h(a)f(con)o(text)f(can)i(b)q(e)f (statically)h(allo)q(cated)g(to)f(eac)o(h)g(pro)q(cedure.)36 b(The)20 b(static)75 102 y(allo)q(cation)h(can)e(b)q(e)h(done)g(in)h(a) e(pream)o(ble,)i(as)e(part)g(of)g(initialization)j(co)q(de.)34 b(Or,)20 b(it)g(can)f(b)q(e)i(done)75 158 y(a)d(compile/link)j(time,)e (if)g(the)g(implemen)o(tation)g(has)g(additional)h(mec)o(hanisms)f(to)e (reserv)o(e)i(con)o(text)75 214 y(v)m(alues.)k(Comm)o(unicators)15 b(to)g(b)q(e)i(used)f(b)o(y)g(the)g(di\013eren)o(t)g(pro)q(cedures)h (can)f(b)q(e)g(built)h(in)g(a)f(pream)o(ble,)75 271 y(if)h(the)g (executing)h(groups)e(are)h(statically)g(de\014ned;)i(if)e(the)g (executing)h(groups)e(c)o(hange)h(dynamically)l(,)75 327 y(then)f(a)g(new)h(comm)o(unicator)e(has)h(to)g(b)q(e)h(built)g (whenev)o(er)g(the)f(executing)h(group)f(c)o(hanges,)g(but)h(this)75 384 y(new)d(comm)o(unicator)g(can)g(b)q(e)h(built)g(using)g(the)f(same) g(t)o(yp)q(e)g(of)g(preallo)q(cated)h(con)o(text.)k(If)14 b(the)g(parallel)75 440 y(pro)q(cedures)k(can)f(b)q(e)g(organized)g(in) o(to)g(libraries,)h(so)f(that)f(only)h(one)g(pro)q(cedure)h(of)e(eac)o (h)h(library)h(can)75 497 y(b)q(e)i(concurren)o(tly)g(activ)o(e)f(in)i (eac)o(h)e(pro)q(cessor,)h(then)f(it)h(is)g(su\016cien)o(t)g(to)e(allo) q(cate)i(one)g(con)o(text)e(p)q(er)75 553 y(library)l(.)75 687 y Fv(P)o(a)o(rallel)c(p)o(ro)q(cedures)i(that)g(a)o(re)f (non-reentrant)i(within)f(each)g(executing)h(group)75 778 y FC(This)k(co)o(v)o(ers)g(the)g(case)f(where,)j(at)d(an)o(y)g(p)q (oin)o(t)i(in)f(time,)h(for)f(eac)o(h)g(pro)q(cess)g(group,)g(there)g (can)g(b)q(e)75 834 y(at)d(most)g(one)h(activ)o(e)g(in)o(v)o(o)q (cation)g(of)g(a)f(parallel)i(pro)q(cedure)g(b)o(y)f(a)f(pro)q(cess)h (mem)o(b)q(er.)31 b(Ho)o(w)o(ev)o(er,)19 b(it)75 891 y(migh)o(t)i(b)q(e)h(p)q(ossible)i(that)c(the)i(same)f(pro)q(cedure)h (is)g(concurren)o(tly)g(in)o(v)o(ok)o(ed)g(in)g(t)o(w)o(o)e(partially)j (\(or)75 947 y(completely\))17 b(o)o(v)o(erlapping)f(groups.)k(F)l(or) 15 b(example,)i(the)e(same)g(collectiv)o(e)j(comm)o(unication)e (function)75 1004 y(ma)o(y)e(b)q(e)i(concurren)o(tly)g(in)o(v)o(ok)o (ed)g(on)f(t)o(w)o(o)f(partially)i(o)o(v)o(erlapping)g(groups.)166 1063 y(In)f(suc)o(h)f(a)g(case,)g(a)f(con)o(text)h(is)g(asso)q(ciated)h (with)f(eac)o(h)g(parallel)i(pro)q(cedure)f(and)f(eac)o(h)g(executing) 75 1119 y(group,)j(so)h(that)e(o)o(v)o(erlapping)j(execution)f(groups)f (ha)o(v)o(e)g(distinct)i(comm)o(unication)f(con)o(texts.)27 b(\(One)75 1176 y(do)q(es)19 b(not)f(need)i(a)f(di\013eren)o(t)f(con)o (text)h(from)f(eac)o(h)h(group;)g(one)g(merely)h(needs)f(a)g (\\coloring")g(of)f(the)75 1232 y(groups,)e(so)g(that)g(one)h(can)g (generate)f(the)h(comm)o(unicators)f(for)g(eac)o(h)g(parallel)i(pro)q (cedure)g(when)f(the)75 1288 y(execution)g(groups)e(are)g(de\014ned.)22 b(Here)16 b(again,)f(one)h(only)g(need)h(one)e(con)o(text)g(for)g(eac)o (h)h(library)l(,)g(if)g(no)75 1345 y(t)o(w)o(o)e(pro)q(cedures)i(from)e (the)i(same)f(library)h(can)f(b)q(e)h(concurren)o(tly)g(activ)o(e)f(in) h(the)f(same)g(group.)166 1404 y(Note)f(that,)g(for)h(collectiv)o(e)h (comm)o(unication)g(libraries,)g(w)o(e)f(do)g(allo)o(w)g(sev)o(eral)g (concurren)o(t)g(in)o(v)o(o-)75 1460 y(cations)h(within)h(the)g(same)e (group:)22 b(a)15 b(broadcast)h(in)h(a)e(group)h(ma)o(y)g(b)q(e)g (started)g(in)h(a)e(pro)q(cess)i(b)q(efore)75 1517 y(the)e(previous)h (broadcast)f(in)h(that)e(group)h(ended)h(at)f(another)g(pro)q(cess.)20 b(In)c(suc)o(h)f(a)g(case,)g(one)g(cannot)75 1573 y(rely)21 b(on)g(con)o(text)g(mec)o(hanisms)g(to)f(disam)o(biguate)i(successiv)o (e)g(in)o(v)o(o)q(cations)f(of)g(the)g(same)f(parallel)75 1630 y(pro)q(cedure)f(within)f(the)g(same)g(group:)24 b(the)18 b(pro)q(cedure)g(needs)h(to)e(b)q(e)h(implemen)o(ted)i(so)d (as)g(to)g(a)o(v)o(oid)75 1686 y(confusion)i(\(bac)o(k)e(masking\).)28 b(F)l(or)17 b(example,)i(for)e(broadcast,)h(one)g(ma)o(y)f(need)i(to)e (carry)h(additional)75 1743 y(information)g(in)g(messages,)f(suc)o(h)h (as)g(the)f(broadcast)g(ro)q(ot,)g(to)g(help)i(in)f(suc)o(h)g(disam)o (biguation;)i(one)75 1799 y(also)15 b(relies)i(on)e(the)g(pair-wise)h (preserv)m(ation)g(of)f(message)g(order)g(b)o(y)g(MPI.)g(With)g(suc)o (h)h(an)f(approac)o(h,)75 1855 y(w)o(e)f(ma)o(y)g(b)q(e)h(gaining)g(p)q (erformance,)f(but)g(w)o(e)h(lose)f(mo)q(dularit)o(y)l(.)21 b(It)14 b(is)h(insu\016cien)o(t)h(to)d(implemen)o(t)j(the)75 1912 y(parallel)h(pro)q(cedure)g(so)e(that)g(it)h(w)o(orks)f(correctly) h(in)g(isolation,)h(when)f(in)o(v)o(ok)o(ed)g(only)g(once;)g(it)g (needs)75 1968 y(to)f(b)q(e)h(implemen)o(ted)i(so)d(that)g(an)o(y)h(n)o (um)o(b)q(er)g(of)f(successiv)o(e)i(in)o(v)o(o)q(cations)f(will)h (execute)g(correctly)l(.)22 b(Of)75 2025 y(course,)15 b(the)g(same)g(approac)o(h)g(can)g(b)q(e)h(used)g(for)e(other)h (parallel)i(libraries.)75 2159 y Fv(W)o(ell-nested)g(pa)o(rallel)d(p)o (ro)q(cedures)75 2249 y FC(Calls)h(of)g(parallel)h(pro)q(cedures)g(are) e(w)o(ell-nested)i(if)g(a)e(new)h(parallel)h(pro)q(cedure)g(is)f(alw)o (a)o(ys)f(in)o(v)o(ok)o(ed)h(in)75 2306 y(a)j(subset)h(of)e(a)h(group)g (executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 2362 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 2421 y(In)h(suc)o(h)f(a)g(case,)g(a)g(new)g (con)o(text)f(needs)i(to)f(b)q(e)h(dynamically)h(allo)q(cated)f(for)e (eac)o(h)h(new)h(in)o(v)o(o)q(ca-)75 2478 y(tion)h(of)f(a)g(parallel)i (pro)q(cedure.)28 b(Ho)o(w)o(ev)o(er,)16 b(a)i(stac)o(k)e(mec)o(hanism) i(can)g(b)q(e)g(used)g(for)f(allo)q(cating)i(new)75 2534 y(con)o(texts.)29 b(Th)o(us,)19 b(a)g(p)q(ossible)h(mec)o(hanism)f(is)g (\014rst)g(to)f(allo)q(cate)h(a)f(large)h(n)o(um)o(b)q(er)g(of)f(con)o (texts)g(\(up)75 2591 y(to)e(the)h(upp)q(er)h(b)q(ound)g(on)f(the)g (depth)g(of)g(nested)g(parallel)h(pro)q(cedure)g(calls\),)g(and)f(then) g(use)g(a)g(lo)q(cal)75 2647 y(stac)o(k)d(managemen)o(t)g(of)g(these)h (con)o(text's)f(on)g(eac)o(h)h(pro)q(cess)g(to)f(create)h(a)f(new)h (comm)o(unicator)g(\(using)75 2704 y Fv(MPI)p 160 2704 14 2 v 16 w(COMM)p 318 2704 V 16 w(MAKE)p FC(\))g(for)f(eac)o(h)i(new)f (in)o(v)o(o)q(cation.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 127 135 127 134 bop 75 -100 a Fs(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FC(127)75 45 y Fv(The)16 b(General)f(case)75 131 y FC(In)22 b(the)g(general)g(case,)h(there)e(ma)o(y)g(b)q(e)h(m)o (ultiple)i(concurren)o(tly)e(activ)o(e)g(in)o(v)o(o)q(cations)g(of)f (the)g(same)75 187 y(parallel)i(pro)q(cedure)g(within)g(the)f(same)f (group;)j(in)o(v)o(o)q(cations)e(ma)o(y)f(not)h(b)q(e)g(w)o (ell-nested.)41 b(A)22 b(new)75 244 y(con)o(text)17 b(need)i(to)e(b)q (e)h(created)g(for)f(eac)o(h)g(in)o(v)o(o)q(cation.)28 b(It)18 b(is)g(the)g(user)g(resp)q(onsibilit)o(y)i(to)d(mak)o(e)g(sure) 75 300 y(that,)k(if)h(t)o(w)o(o)d(distinct)j(parallel)h(pro)q(cedures)e (are)g(in)o(v)o(ok)o(ed)g(concurren)o(tly)h(on)e(o)o(v)o(erlapping)i (sets)e(of)75 357 y(pro)q(cesses,)15 b(then)h(con)o(text)e(allo)q (cation)j(or)d(comm)o(unicator)h(creation)h(is)f(prop)q(erly)h(co)q (ordinated.)75 499 y Ft(5.8)59 b(Motivating)19 b(Examples)75 602 y Fp(5.8.1)49 b(Current)15 b(Practice)g(#1)75 688 y FC(Example)h(#1a:)147 776 y Fo(main\(int)22 b(argc,)i(char)f (**argv\))147 833 y({)194 889 y(int)h(me,)f(size;)194 946 y(...)194 1002 y(MPI_Init)g(\()h(&argc,)f(argv)g(\);)194 1059 y(MPI_Comm_rank)f(\(MPI_COMM_WORLD,)g(&me\);)194 1115 y(MPI_Comm_size)g(\(MPI_COMM_WORLD,)g(&size\);)194 1228 y(\(void\)printf)h(\("Process)f(\045d)i(size)f(\045d\\n",)g(me,)h (size\);)194 1285 y(...)194 1341 y(MPI_Finalize\(\);)147 1397 y(})75 1486 y FC(Example)19 b(#1a)f(is)h(a)f(do-nothing)h(program) e(that)h(initializes)k(itself)d(legally)l(,)i(and)d(refers)g(to)g(the)h (the)75 1542 y(\\all")f(comm)o(unicator,)g(and)g(prin)o(ts)g(a)f (message.)27 b(It)18 b(terminates)g(itself)h(legally)g(to)q(o.)27 b(This)18 b(example)75 1599 y(do)q(es)e(not)e(imply)j(that)d(MPI)i (supp)q(orts)f Fo(printf)p FC(-lik)o(e)g(comm)o(unication)h(itself.)75 1655 y(Example)g(#1b)f(\(supp)q(osing)h(that)e Fo(size)h FC(is)h(ev)o(en\):)170 1744 y Fo(main\(int)23 b(argc,)g(char)h (**argv\))170 1800 y({)242 1857 y(int)f(me,)h(size;)242 1913 y(int)f(SOME_TAG)g(=)h(0;)242 1970 y(...)242 2026 y(MPI_Init\(argc,)e(argv\);)242 2139 y(MPI_Comm_rank\(MPI_COMM_WOR)o (LD,)f(&me\);)71 b(/*)23 b(local)h(*/)242 2195 y (MPI_Comm_size\(MPI_COMM_WOR)o(LD,)d(&size\);)i(/*)g(local)h(*/)242 2308 y(if\(\(me)f(\045)h(2\))f(==)h(0\))242 2365 y({)314 2421 y(/*)f(send)h(unless)f(highest-numbered)e(process)i(*/)314 2478 y(if\(\(me)g(+)g(1\))h(<)g(size\))385 2534 y(MPI_Send\(...,)e(me)i (+)g(1,)f(SOME_TAG,)g(MPI_COMM_WORLD\);)242 2591 y(})242 2647 y(else)314 2704 y(MPI_Recv\(...,)f(me)h(-)h(1,)g(SOME_TAG,)e (MPI_COMM_WORLD\);)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 128 136 128 135 bop 75 -100 a FC(128)123 b Fs(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)242 102 y Fo(...)242 158 y(MPI_Finalize\(\);)170 214 y(})75 310 y FC(Example)h(#1b)f(sc)o(hematically)i(illustrates)g(message)e (exc)o(hanges)h(b)q(et)o(w)o(een)g(\\ev)o(en")f(and)h(\\o)q(dd")f(pro-) 75 367 y(cesses)h(in)g(the)f(\\all")g(comm)o(unicator.)75 491 y Fp(5.8.2)49 b(Current)15 b(Practice)g(#2)147 578 y Fo(main\(int)22 b(argc,)i(char)f(**argv\))147 634 y({)194 690 y(int)h(me,)f(count;)194 747 y(void)h(*data;)194 803 y(...)194 916 y(MPI_Init\(argc,)e(argv\);)194 973 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&me\);)194 1086 y(if\(me)i(==)h(0\))194 1142 y({)290 1199 y(/*)f(get)h(input,)f(create) g(buffer)g(``data'')g(*/)290 1255 y(...)194 1311 y(})194 1424 y(MPI_Bcast\(data,)f(count,)h(MPI_BYTE,)g(0,)g(MPI_COMM_WORLD\);) 194 1537 y(...)194 1650 y(MPI_Finalize\(\);)147 1707 y(})75 1815 y FC(This)16 b(example)g(illustrates)g(the)f(use)h(of)f(a)g (collectiv)o(e)h(comm)o(unication.)75 1939 y Fp(5.8.3)49 b(\(App)o(ro)o(ximate\))16 b(Current)f(Practice)g(#3)123 2026 y Fo(main\(int)22 b(argc,)i(char)f(**argv\))123 2083 y({)170 2139 y(int)h(me,)f(count,)g(count2;)170 2195 y(void)h(*send_buf,)e(*recv_buf,)h(*send_buf2,)f(*recv_buf2;)170 2252 y(MPI_Group)h(MPI_GROUP_WORLD,)f(grp0,)h(grprem;)170 2308 y(MPI_Comm)g(commslave;)170 2365 y(static)g(int)h(ranks[])f(=)h ({0};)170 2421 y(...)170 2478 y(MPI_Init\(argc,)e(argv\);)170 2534 y(MPI_Comm_group\(MPI_COMM_WORLD)o(,)f(&MPI_GROUP_WORLD\);)170 2591 y(MPI_Comm_rank\(MPI_COMM_WORLD,)f(&me\);)47 b(/*)24 b(local)f(*/)170 2704 y(MPI_Group_excl\(MPI_GROUP_WORL)o(D,1,)e(ranks,) i(&grprem\);)46 b(/*)24 b(local)f(*/)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 129 137 129 136 bop 75 -100 a Fs(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FC(129)170 45 y Fo(MPI_Comm_make\(MPI_COMM_WORLD,)20 b(grprem,)j(&commslave\);)170 158 y(if\(me)h(!=)f(0\))170 214 y({)218 271 y(/*)h(compute)f(on)g(slave)g(*/)218 327 y(...)218 384 y(MPI_Reduce\(send_buf,recv_bu)o(ff,count)o(,)e (MPI_INT,)i(MPI_SUM,)g(1,)g(commslave\);)218 440 y(...)170 497 y(})170 553 y(/*)h(zero)f(falls)h(through)e(immediately)h(to)g (this)h(reduce,)f(others)g(do)g(later...)g(*/)170 610 y(MPI_Reduce\(send_buf2,)e(recv_buff2,)i(count2,)433 666 y(MPI_INT,)g(MPI_SUM,)f(0,)i(MPI_COMM_WORLD\);)170 723 y(MPI_Finalize\(\);)123 779 y(})75 881 y FC(This)18 b(example)h(illustrates)g(ho)o(w)e(a)h(group)f(consisting)i(of)e(all)i (but)f(the)g(zeroth)f(pro)q(cess)h(of)g(the)g(\\all")75 938 y(group)13 b(is)h(created,)f(and)h(then)g(ho)o(w)f(a)g(comm)o (unicator)g(is)h(formed)f(\()g Fv(commslave)p FC(\))e(for)i(that)g(new) h(group.)75 994 y(The)f(new)h(comm)o(unicator)e(is)i(used)g(in)g(a)e (collectiv)o(e)j(call,)g(and)e(all)h(pro)q(cesses)f(execute)h(a)f (collectiv)o(e)i(call)75 1051 y(in)j(the)33 b Fl(MPI)p 303 1051 13 2 v 15 w(COMM)p 448 1051 V 14 w(W)o(ORLD)18 b FC(con)o(text.)25 b(This)18 b(example)h(illustrates)f(ho)o(w)f(the)g (t)o(w)o(o)f(comm)o(unicators)75 1107 y(\(that)c(inheren)o(tly)k(p)q (ossess)d(distinct)i(con)o(texts\))d(protect)h(comm)o(unication.)20 b(That)13 b(is,)h(comm)o(unication)75 1164 y(in)30 b Fl(MPI)p 219 1164 V 14 w(COMM)p 363 1164 V 15 w(W)o(ORLD)15 b FC(is)h(insulated)g(from)f(comm)o(unication)h(in)31 b Fv(commslave)p FC(,)13 b(and)i(vice)i(v)o(ersa.)166 1220 y(In)h(summary)l(,)g(\\group)f(safet)o(y")f(is)i(ac)o(hiev)o(ed)h (via)f(comm)o(unicators)f(b)q(ecause)i(distinct)g(con)o(texts)75 1277 y(within)d(comm)o(unicators)f(are)g(enforced)h(to)e(b)q(e)i (unique)h(on)e(an)o(y)g(pro)q(cess.)75 1398 y Fp(5.8.4)49 b(Example)17 b(#4)75 1484 y FC(The)e(follo)o(wing)g(example)g(is)g (mean)o(t)f(to)f(illustrate)j(\\safet)o(y")d(b)q(et)o(w)o(een)h(p)q (oin)o(t-to-p)q(oin)o(t)i(and)e(collectiv)o(e)75 1540 y(comm)o(unication.)20 b(MPI)12 b(guaran)o(tees)g(that)g(a)g(single)i (comm)o(unicator)e(can)h(do)f(safe)g(p)q(oin)o(t-to-p)q(oin)o(t)i(and) 75 1596 y(collectiv)o(e)j(comm)o(unication.)147 1687 y Fo(#define)22 b(TAG_ARBITRARY)h(12345)147 1744 y(#define)f (SOME_COUNT)166 b(50)147 1857 y(main\(int)22 b(argc,)i(char)f(**argv\)) 147 1913 y({)194 1970 y(int)h(me;)194 2026 y(MPI_Group)f (MPI_GROUP_WORLD,)f(subgroup;)194 2083 y(int)i(ranks[])f(=)g({2,)h(4,)f (6,)h(8};)194 2139 y(...)194 2195 y(MPI_Init\(argc,)e(argv\);)194 2252 y(MPI_Comm_group\(MPI_COMM_WORL)o(D,)f(&MPI_GROUP_WORLD\);)194 2365 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g(4,)j(ranks,)f (&subgroup\);)f(/*)i(local)f(*/)194 2421 y(MPI_Group_rank\(subgroup,)e (&me\);)119 b(/*)23 b(local)g(*/)194 2534 y (MPI_Comm_make\(MPI_COMM_WORLD)o(,)e(subgroup,)i(&the_comm\);)194 2647 y(if\(me)g(!=)h(MPI_UNDEFINED\))194 2704 y({)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 130 138 130 137 bop 75 -100 a FC(130)123 b Fs(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)290 45 y Fo(MPI_Irecv\(buff1,)21 b(count,)i(MPI_DOUBLE,)g(MPI_ANY_SOURCE,)f (TAG_ARBITRARY,)719 102 y(the_comm,)h(request\))290 158 y(MPI_Isend\(buff2,)e(count,)i(MPI_DOUBLE,)g(\(me+1\)\0454,)f (TAG_ARBITRARY,)719 214 y(the_comm,)h(request+1\))194 271 y(})194 384 y(for\(i)g(=)h(0;)g(i)g(<)f(SOME_COUNT,)g(i++\))242 440 y(MPI_Reduce\(...,)f(the_comm\);)194 497 y(MPI_Waitall\(2,)g (request,)h(status\))194 553 y(MPI_Finalize\(\);)147 610 y(})75 738 y Fp(5.8.5)49 b(Lib)o(ra)o(ry)16 b(Example)h(#1)75 826 y FC(The)e(main)h(program:)147 925 y Fo(main\(int)22 b(argc,)i(char)f(**argv\))147 982 y({)194 1038 y(int)h(done)f(=)h(0;) 194 1095 y(user_lib_t)f(*libh_a,)g(*libh_b;)194 1151 y(void)h(*dataset1,)e(*dataset2;)194 1207 y(...)194 1264 y(MPI_Init\(argc,)g(argv\);)194 1320 y(...)194 1377 y (init_user_lib\(MPI_COMM_WORLD)o(,)f(&libh_a\);)194 1433 y(init_user_lib\(MPI_COMM_WORLD)o(,)g(&libh_b\);)194 1490 y(...)194 1546 y(user_start_op\(libh_a,)g(dataset1\);)194 1603 y(user_start_op\(libh_b,)g(dataset2\);)194 1659 y(...)194 1716 y(while\(!done\))194 1772 y({)266 1828 y(/*)i(work)h(*/)266 1885 y(...)266 1941 y(MPI_Reduce\(...,)e (MPI_COMM_WORLD\);)266 1998 y(...)266 2054 y(/*)h(see)h(if)g(done)f(*/) 266 2111 y(...)194 2167 y(})194 2224 y(user_end_op\(libh_a\);)194 2280 y(user_end_op\(libh_b\);)194 2337 y(MPI_Finalize\(\);)147 2393 y(})75 2491 y FC(The)15 b(user)h(library)g(initialization)i(co)q (de:)147 2591 y Fo(void)23 b(init_user_lib\(MPI_Comm)e(*comm,)i (user_lib_t)f(**handle\))147 2647 y({)194 2704 y(user_lib_t)h(*save;) 1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 131 139 131 138 bop 75 -100 a Fs(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FC(131)194 102 y Fo(user_lib_initsave\(&save\);)21 b(/*)i(local)h(*/)194 158 y(MPI_Comm_dup\(comm,)e(&\(save)h(->)g (comm\)\);)194 271 y(/*)h(other)f(inits)g(*/)194 327 y(...)194 440 y(*handle)g(=)h(save;)147 497 y(})75 591 y FC(User)15 b(start-up)g(co)q(de:)147 686 y Fo(void)23 b(user_start_op\(user_lib_t)d(*handle,)j(void)h(*data\))147 743 y({)194 799 y(MPI_Irecv\()f(...,)g(handle->comm,)f(&\(handle)h(->)h (irecv_handle\))e(\);)194 855 y(MPI_Isend\()h(...,)g(handle->comm,)f (&\(handle)h(->)h(isend_handle\))e(\);)147 912 y(})75 1007 y FC(User)15 b(clean-up)i(co)q(de:)147 1101 y Fo(void)23 b(user_end_op\(user_lib_t)e(*handle\))147 1158 y({)194 1214 y(MPI_Status)i(*status;)194 1271 y(MPI_Wait\(handle)f(->)i (isend_handle,)e(status\);)194 1327 y(MPI_Wait\(handle)g(->)i (irecv_handle,)e(status\);)147 1384 y(})75 1506 y Fp(5.8.6)49 b(Lib)o(ra)o(ry)16 b(Example)h(#2)75 1593 y FC(The)e(main)h(program:) 147 1687 y Fo(main\(int)22 b(argc,)i(char)f(**argv\))147 1744 y({)194 1800 y(int)h(ma,)f(mb;)194 1857 y(...)194 1970 y(static)g(int)h(list_a[])f(=)g({0,)h(1};)75 2026 y(#if)47 b(defined\(EXAMPLE_2B\))22 b(|)h(defined\(EXAMPLE_2C\))194 2083 y(static)g(int)h(list_b[])f(=)g({0,)h(2)f(,3};)75 2139 y(#else/*)g(EXAMPLE_2A)f(*/)194 2195 y(static)h(int)h(list_b[])f (=)g({0,)h(2};)75 2252 y(#endif)194 2308 y(int)g(size_list_a)e(=)i (sizeof\(list_a\)/sizeof\(int)o(\);)194 2365 y(int)g(size_list_b)e(=)i (sizeof\(list_b\)/sizeof\(int)o(\);)194 2478 y(...)194 2534 y(MPI_Init\(argc,)e(argv\);)194 2591 y(...)194 2704 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)f(size_list_a,)h(list_a,)h (&group_a\);)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 132 140 132 139 bop 75 -100 a FC(132)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)194 45 y Fo(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)21 b(size_list_b,)h (list_b,)h(&group_b\);)194 158 y(MPI_Comm_make\(MPI_COMM_WORLD)o(,)e (group_a,)i(&comm_a\);)194 214 y(MPI_Comm_make\(MPI_COMM_WORLD)o(,)e (group_b,)i(&comm_b\);)194 327 y(MPI_Comm_rank\(comm_a,)e(&ma\);)194 384 y(MPI_Comm_rank\(comm_b,)g(&mb\);)194 497 y(if\(ma)i(!=)h (MPI_UNDEFINED\))266 553 y(lib_call\(comm_a\);)194 666 y(if\(mb)f(!=)h(MPI_UNDEFINED\))194 723 y({)242 779 y (lib_call\(comm_b\);)242 835 y(lib_call\(comm_b\);)194 892 y(})194 1005 y(MPI_Finalize\(\);)147 1061 y(})75 1157 y FC(The)15 b(library:)147 1253 y Fo(void)23 b(lib_call\(MPI_Comm) e(comm\))147 1309 y({)194 1366 y(int)j(me,)f(done)h(=)f(0;)194 1422 y(MPI_Comm_rank\(comm,)f(&me\);)194 1479 y(if\(me)h(==)h(0\))266 1535 y(while\(!done\))266 1592 y({)337 1648 y(MPI_Recv\(...,)f (MPI_ANY_SOURCE,)e(MPI_ANY_TAG,)i(comm\);)337 1705 y(...)266 1761 y(})194 1818 y(else)194 1874 y({)242 1930 y(/*)h(work)f(*/)242 1987 y(MPI_Send\(...,)f(0,)i(ARBITRARY_TAG,)e(comm\);)242 2043 y(....)194 2100 y(})75 2156 y(#ifdef)h(EXAMPLE_2C)194 2213 y(/*)h(include)f(\(resp,)g(exclude\))g(for)g(safety)g(\(resp,)g (no)h(safety\):)f(*/)194 2269 y(MPI_Sync\(comm\);)75 2326 y(#endif)147 2382 y(})75 2478 y FC(The)e(ab)q(o)o(v)o(e)f(example) h(is)g(really)g(three)g(examples,)h(dep)q(ending)g(on)f(whether)f(or)g (not)g(one)h(includes)75 2534 y(rank)f(3)h(in)g Fv(list)p 345 2534 14 2 v 17 w(b)p FC(,)h(and)f(whether)g(or)f(not)g(a)g(sync)o (hronize)i(is)f(included)i(in)f Fv(lib)p 1477 2534 V 16 w(call)p FC(.)36 b(This)21 b(example)75 2591 y(illustrates)14 b(that,)e(despite)j(con)o(texts,)d(subsequen)o(t)i(calls)g(to)e Fv(lib)p 1166 2591 V 17 w(call)h FC(with)g(the)g(same)g(con)o(text)f (need)i(not)75 2647 y(b)q(e)h(safe)e(from)g(one)i(another)e(\(collo)q (quially)l(,)k(\\bac)o(k-masking"\).)i(Safet)o(y)13 b(is)h(realized)i (if)e(the)28 b Fv(MPI)p 1772 2647 V 16 w(Sync)75 2704 y FC(is)17 b(added.)25 b(What)16 b(this)h(demonstrates)f(is)h(that)f (libraries)i(ha)o(v)o(e)e(to)g(b)q(e)h(written)g(carefully)l(,)h(ev)o (en)f(with)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 133 141 133 140 bop 75 -100 a Fs(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FC(133)75 45 y(con)o(texts.)23 b(When)17 b(rank)f(3)g(is)h(excluded,)i(then)d(the)h(sync)o(hronize)g(is)g(not)g (needed)g(to)f(get)g(safet)o(y)g(from)75 102 y(bac)o(k)f(masking.)166 158 y(Algorithms)d(lik)o(e)i(\\com)o(bine")e(ha)o(v)o(e)g(strong)f (enough)i(source)f(selectivit)o(y)h(so)f(that)g(they)g(are)g(inher-)75 215 y(en)o(tly)k(ok)m(a)o(y)l(.)22 b(So)16 b(are)f(m)o(ultiple)j(calls) f(to)e(a)h(t)o(ypical)g(tree-broadcast)g(algorithm)f(with)i(the)f(same) f(ro)q(ot.)75 271 y(Ho)o(w)o(ev)o(er,)21 b(m)o(ultiple)i(calls)f(to)f (a)g(t)o(ypical)h(tree-broadcast)e(algorithm)h(|)h(with)f(di\013eren)o (t)g(ro)q(ots)g(|)75 328 y(could)16 b(bac)o(k-mask,)f(pro)q(ducing)i (undesirable)g(results.)k(Therefore,)15 b(suc)o(h)h(algorithms)f(w)o (ould)h(ha)o(v)o(e)f(to)75 384 y(utilize)f(the)e(monotonically)h (increasing)g(tags)e(\(with)g(comm)o(unicator)h(scop)q(e\))g(to)f(k)o (eep)h(things)g(straigh)o(t.)75 441 y(All)19 b(of)e(the)h(foregoing)f (is)h(a)f(supp)q(osition)i(of)f(\\collectiv)o(e)h(calls")f(implemen)o (ted)h(with)f(p)q(oin)o(t-to-p)q(oin)o(t)75 497 y(op)q(erations.)25 b(MPI)17 b(implemen)o(tations)h(ma)o(y)e(or)g(ma)o(y)h(not)f(implemen)o (t)i(collectiv)o(e)h(calls)f(using)f(p)q(oin)o(t-)75 554 y(to-p)q(oin)o(t)h(op)q(erations.)28 b(These)19 b(algorithms)f(are) f(used)i(to)e(illustrate)i(the)g(issues)f(of)g(correctness)g(and)75 610 y(safet)o(y)l(,)c(indep)q(enden)o(t)k(of)c(ho)o(w)h(MPI)g(implemen) o(ts)i(its)e(collectiv)o(e)i(calls.)75 734 y Fp(5.8.7)49 b(Inter-Communication)16 b(Examples)75 820 y Fv(Example)f(1:)k (Three-Group)e(\\Pip)q(eline")409 963 y Fo(+---------+)213 b(+---------+)h(+---------+)409 1019 y(|)h(|)f(|)h(|)g(|)f(|)409 1076 y(|)24 b(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f (|)g(Group)h(2)f(|)409 1132 y(|)215 b(|)f(|)h(|)g(|)f(|)409 1189 y(+---------+)f(+---------+)h(+---------+)75 1353 y FC(Groups)16 b(0)g(and)h(1)f(comm)o(unicate.)24 b(Groups)16 b(1)h(and)f(2)g(comm)o(unicate.)24 b(Therefore,)17 b(group)f(0)g (requires)75 1410 y(one)f(in)o(ter-comm)o(unicator,)f(group)g(1)g (requires)i(t)o(w)o(o)d(in)o(ter-comm)o(unicators,)h(and)h(group)f(2)g (requires)h(1)75 1466 y(in)o(ter-comm)o(unicator.)147 1574 y Fo(main\(int)22 b(argc,)i(char)f(**argv\))147 1631 y({)194 1687 y(MPI_Comm)g(myComm;)166 b(/*)24 b (intra-communicator)d(of)j(local)f(sub-group)g(*/)194 1744 y(MPI_Comm)g(myFirstComm;)46 b(/*)24 b(inter-communicator)d(*/)194 1800 y(MPI_Comm)i(mySecondComm;)f(/*)i(second)f(inter-communicator)e (\(group)i(1)h(only\))f(*/)194 1857 y(MPI_Intercomm_request)e (inter_request1,)h(inter_request2;)194 1913 y(int)i(membershipKey;)194 1970 y(int)g(rank;)194 2083 y(MPI_Init\(argc,)e(argv\);)194 2139 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&rank\);)194 2252 y(/*)j(User)f(code)h(must)f(generate)g(membershipKey)f(in)h(the)h (range)f([0,)g(1,)h(2])g(*/)194 2308 y(membershipKey)e(=)i(rank)f(\045) h(3;)194 2421 y(/*)g(Build)f(intra-communicator)e(for)j(local)f (sub-group)g(*/)194 2478 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e (membershipKey,)h(rank,)h(&myComm\);)194 2591 y(/*)h(Build)f (inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 2647 y(if)i(\(membershipKey)e(==)h(0\))194 2704 y({)501 b(/*)24 b(Group)f(0)h(communicates)e(with)h(group)h(1.)f(*/)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 134 142 134 141 bop 75 -100 a FC(134)123 b Fs(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)242 45 y Fo(MPI_Intercomm_start\()21 b(myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,) 743 102 y(1,)j(&inter_request1)e(\);)242 158 y(MPI_Intercomm_finish\()f (inter_request1,)h(myFirstComm)g(\);)194 214 y(})194 271 y(else)i(if)f(\(membershipKey)f(==)i(1\))194 327 y({)334 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)g (2.)f(*/)242 384 y(MPI_Intercomm_start\()e(myComm,)i(0,)h (MPI_COMM_WORLD,)d(0,)743 440 y(1,)j(&inter_request1)e(\);)242 497 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 553 y(12,)j(&inter_request2)d(\);)242 610 y(MPI_Intercomm_finish\()g (inter_request1,)h(myFirstComm)g(\);)242 666 y(MPI_Intercomm_finish\()f (inter_request2,)h(mySecondComm)g(\);)194 723 y(})194 779 y(else)i(if)f(\(membershipKey)f(==)i(2\))194 835 y({)501 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)h(1.)f(*/)242 892 y(MPI_Intercomm_start\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 948 y(12,)j(&inter_request1)d(\);)242 1005 y(MPI_Intercomm_finish\()g (inter_request1,)h(myFirstComm)g(\);)194 1061 y(})194 1174 y(/*)i(Do)f(work)h(...)f(*/)194 1287 y(MPI_Finalize\(\);)147 1344 y(})75 1462 y Fv(Example)15 b(2:)k(Three-Group)e(\\Ring")290 1548 y Fo(+-----------------------)o(--------)o(-------)o(-------)o (--------)o(------+)290 1605 y(|)1408 b(|)290 1661 y(|)95 b(+---------+)213 b(+---------+)h(+---------+)94 b(|)290 1718 y(|)h(|)215 b(|)f(|)h(|)g(|)f(|)96 b(|)290 1774 y(+-->)23 b(|)h(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f (|)g(Group)h(2)f(|)h(<--+)409 1831 y(|)215 b(|)f(|)h(|)g(|)f(|)409 1887 y(+---------+)f(+---------+)h(+---------+)166 1985 y FC(Groups)14 b(0)g(and)h(1)f(comm)o(unicate.)20 b(Groups)14 b(1)g(and)h(2)f(comm)o(unicate.)20 b(Groups)14 b(0)g(and)h(2)f(comm)o (u-)75 2041 y(nicate.)21 b(Therefore,)14 b(eac)o(h)h(requires)h(t)o(w)o (o)e(in)o(ter-comm)o(unicators.)147 2139 y Fo(main\(int)22 b(argc,)i(char)f(**argv\))147 2195 y({)194 2252 y(MPI_Comm)47 b(myComm;)118 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)g (*/)194 2308 y(MPI_Comm)g(myFirstComm;)f(/*)i(inter-communicators)d(*/) 194 2365 y(MPI_Comm)i(mySecondComm;)194 2421 y(MPI_Intercomm_request)e (inter_request1,)h(inter_request2;)194 2478 y(int)i(membershipKey;)194 2534 y(int)g(rank;)194 2647 y(MPI_Init\(argc,)e(argv\);)194 2704 y(MPI_Comm_rank\(MPI_COMMM_WORL)o(D,)f(&rank\);)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 135 143 135 142 bop 75 -100 a Fs(5.8.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b FC(135)194 45 y Fo(...)194 158 y(/*)24 b(User)f(code)h(must)f(generate)g(membershipKey)f(in)h(the)h(range)f ([0,)g(1,)h(2])g(*/)194 214 y(membershipKey)e(=)i(rank)f(\045)h(3;)194 327 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/) 194 384 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e(membershipKey,)h(rank,)h (&myComm\);)194 497 y(/*)h(Build)f(inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 553 y(if)i(\(membershipKey)e(==) h(0\))194 610 y({)310 b(/*)24 b(Group)f(0)h(communicates)e(with)h (groups)g(1)h(and)g(2.)f(*/)242 666 y(MPI_Intercomm_start\()e(myComm,)i (0,)h(MPI_COMM_WORLD,)d(1,)743 723 y(1,)j(&inter_request1)e(\);)242 779 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 835 y(2,)j(&inter_request2)e(\);)194 892 y(})194 948 y(else)i(if)f(\(membershipKey)f(==)i(1\))194 1005 y({)215 b(/*)24 b(Group)f(1)g(communicates)g(with)g(groups)g(0)h(and)f(2.)h(*/) 242 1061 y(MPI_Intercomm_start\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,) 743 1118 y(1,)j(&inter_request1)e(\);)242 1174 y(MPI_Intercomm_start\() f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 1231 y(12,)j(&inter_request2) d(\);)194 1287 y(})194 1344 y(else)j(if)f(\(membershipKey)f(==)i(2\)) 194 1400 y({)191 b(/*)24 b(Group)f(2)h(communicates)e(with)h(groups)g (0)h(and)f(1.)h(*/)242 1456 y(MPI_Intercomm_start\()d(myComm,)i(0,)h (MPI_COMM_WORLD,)d(0,)743 1513 y(2,)j(&inter_request1)e(\);)242 1569 y(MPI_Intercomm_start\()f(myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 1626 y(12,)j(&inter_request2)d(\);)194 1682 y(})194 1795 y(/*)j(Everyone)f(has)g(the)h(same)f("FINISH")g(code...)g(*/)194 1852 y(MPI_Intercomm_finish\(inter_r)o(equest1,)d(&myFirstComm\);)194 1908 y(MPI_Intercomm_finish\(inter_r)o(equest2,)g(&mySecondComm\);)194 2021 y(/*)k(Do)f(some)h(work)f(...)h(*/)194 2134 y(MPI_Finalize\(\);) 147 2190 y(})75 2371 y Fv(Example)15 b(3:)k(Three-Group)e(\\Pip)q (eline")e(Using)h(Name)f(Service)409 2478 y Fo(+---------+)213 b(+---------+)h(+---------+)409 2534 y(|)h(|)f(|)h(|)g(|)f(|)409 2591 y(|)24 b(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f (|)g(Group)h(2)f(|)409 2647 y(|)215 b(|)f(|)h(|)g(|)f(|)409 2704 y(+---------+)f(+---------+)h(+---------+)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 136 144 136 143 bop 75 -100 a FC(136)123 b Fs(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)166 45 y FC(Groups)21 b(0)g(and)h(1)f(comm)o(unicate.)39 b(Groups)21 b(1)g(and)h(2)f(comm)o(unicate.)39 b(Therefore,)23 b(group)e(0)75 102 y(requires)e(one)f(in)o(ter-comm)o(unicator,)h (group)e(1)h(requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators,)h(and)g (group)g(2)75 158 y(requires)e(1)f(in)o(ter-comm)o(unicator.)147 276 y Fo(main\(int)22 b(argc,)i(char)f(**argv\))147 332 y({)194 389 y(MPI_Comm)g(myComm;)142 b(/*)24 b(intra-communicator)d(of) j(local)f(sub-group)g(*/)194 445 y(MPI_Comm)g(myFirstComm;)f(/*)i (inter-communicator)d(*/)194 502 y(MPI_Comm)i(mySecondComm;/*)f(second) h(inter-communicator)e(\(group)i(1)h(only\))f(*/)194 558 y(MPI_Intercomm_request)e(inter_request1,)h(inter_request2;)194 615 y(int)i(color,)f(rank,)g(size;)194 728 y(MPI_Init\(argc,)f(argv\);) 194 784 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&rank\);)194 841 y(MPI_Comm_size\(MPI_COMM_WORLD)o(,)g(&size\);)194 897 y(...)194 1010 y(/*)j(User)f(code)h(must)f(generate)g(color)g(in)g (the)h(range)f([0,)g(1,)h(2])g(*/)194 1066 y(if)g(\()g(rank)f(<)h (\(\(size+1\)/3\))e(\))242 1123 y(color)h(=)h(0;)194 1179 y(else)g(if)f(\()h(rank)f(<)h(\(2*\(size+1\)/3\))e(\))242 1236 y(color)h(=)h(1;)194 1292 y(else)242 1349 y(color)f(=)h(2;)194 1462 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/) 194 1518 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e(color,)i(rank,)g (&myComm\);)194 1631 y(/*)h(Build)f(inter-communicators.)45 b(Group)23 b(membership)g(conditions)f(must)i(be)f(*/)194 1687 y(/*)h(provided)f(by)g(the)h(user.)f(*/)194 1744 y(if)h(\(color)f(==)h(0\))194 1800 y({)167 b(/*)24 b(Group)f(0)h (communicates)e(with)h(group)g(1.)h(*/)242 1857 y (MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(01",)i (&inter_request1\);)242 1913 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)194 1970 y(})194 2026 y(else)i(if)f(\(color)g(==)h(1\))194 2083 y({)167 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)f(2.)h(*/) 242 2139 y(MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(01",)i (&inter_request1\);)242 2195 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(12",)i(&inter_request2\);)242 2252 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)242 2308 y(MPI_Intercomm_finish)f (\(inter_request2,)h(&mySecondComm\);)194 2365 y(})194 2421 y(else)i(if)f(\(color)g(==)h(2\))194 2478 y({)167 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)g(1.)h(*/)242 2534 y(MPI_Intercomm_name_start\(m)o(yComm,)d("Connect)h(12",)i (&inter_request1\);)242 2591 y(MPI_Intercomm_finish)d (\(inter_request1,)h(&myFirstComm\);)194 2647 y(})1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 137 145 137 144 bop 75 -100 a Fs(5.8.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b FC(137)194 45 y Fo(/*)24 b(Do)f(some)h(work)f(...)h (*/)194 158 y(MPI_Finalize\(\);)147 214 y(})75 333 y Fv(Example)15 b(4:)k(Three-Group)e(\\Ring")e(Using)g(Name)g(Service)290 419 y Fo(+-----------------------)o(--------)o(-------)o(-------)o (--------)o(------+)290 476 y(|)1408 b(|)290 532 y(|)95 b(+---------+)213 b(+---------+)h(+---------+)94 b(|)290 589 y(|)h(|)215 b(|)f(|)h(|)g(|)f(|)96 b(|)290 645 y(+-->)23 b(|)h(Group)f(0)h(|)f(<----->)g(|)h(Group)f(1)h(|)g(<----->)f(|)g (Group)h(2)f(|)h(<--+)409 701 y(|)215 b(|)f(|)h(|)g(|)f(|)409 758 y(+---------+)f(+---------+)h(+---------+)75 856 y FC(Groups)12 b(0)g(and)h(1)f(comm)o(unicate.)19 b(Groups)13 b(1)f(and)g(2)h(comm)o(unicate.)19 b(Groups)12 b(0)g(and)h(2)f(comm)o (unicate.)75 912 y(Therefore,)j(eac)o(h)g(requires)h(t)o(w)o(o)e(in)o (ter-comm)o(unicators.)147 1010 y Fo(main\(int)22 b(argc,)i(char)f (**argv\))147 1066 y({)194 1123 y(MPI_Comm)47 b(myComm;)71 b(/*)23 b(intra-communicator)f(of)h(local)g(sub-group)g(*/)194 1179 y(MPI_Comm)47 b(myFirstComm;)380 b(/*)24 b(inter-communicators)d (*/)194 1236 y(MPI_Comm)47 b(mySecondComm;)194 1292 y (MPI_Intercomm_request)21 b(inter_request1,)h(inter_request2;)194 1349 y(int)i(color,)f(rank,)g(size;)194 1462 y(MPI_Init\(argc,)f (argv\);)194 1518 y(MPI_Comm_size\(MPI_COMM_WORLD)o(,)f(&size\);)194 1574 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)g(&rank\);)194 1631 y(...)194 1744 y(/*)j(User)f(code)h(must)f(generate)g(color)g(in)g (the)h(range)f([0,)g(1,)h(2])g(*/)194 1800 y(if)g(\()g(rank)f(==)h(0)f (\))242 1857 y(color)g(=)h(0;)194 1913 y(else)g(if)f(\()h(rank)f(==)h (1)g(\))242 1970 y(color)f(=)h(1;)194 2026 y(else)242 2083 y(color)f(=)h(2;)194 2195 y(/*)g(Build)f(intra-communicator)e(for) j(local)f(sub-group)g(*/)194 2252 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,) e(color,)i(rank,)g(&myComm\);)194 2365 y(/*)h(Build)f (inter-communicators.)45 b(Group)23 b(membership)g(conditions)f(must)i (be)f(*/)194 2421 y(/*)h(provided)f(by)g(the)h(user.)f(*/)194 2478 y(if)h(\(color)f(==)h(0\))194 2534 y({)119 b(/*)24 b(Group)f(0)h(communicates)e(with)i(groups)f(1)g(and)h(2.)f(*/)242 2591 y(MPI_Intercomm_name_start\(m)o(yComm,)e("Connect)h(01",)i (&inter_request1\);)242 2647 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(02",)i(&inter_request2\);)194 2704 y(})-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 138 146 138 145 bop 75 -100 a FC(138)118 b Fs(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(COMMUNICA)l(TORS,)h(AND)f(CA)o(CHING)194 45 y Fo(else)24 b(if)f(\(color)g(==)h(1\))194 102 y({)119 b(/*)24 b(Group)f(1)h(communicates)e(with)i(groups)f(0)g(and)h(2.)f(*/) 242 158 y(MPI_Intercomm_name_start\(m)o(yComm,)e("Connect)h(01",)i (&inter_request1\);)242 214 y(MPI_Intercomm_name_start\(m)o(yComm,)d ("Connect)h(12",)i(&inter_request2\);)194 271 y(})194 327 y(else)g(if)f(\(\))194 384 y({)96 b(/*)23 b(Group)g(2)h(communicates)e(with)i(groups)f(0)g(and)h (1.)f(*/)242 440 y(MPI_Intercomm_name_start\(m)o(yComm,)e("Connect)h (02",)i(&inter_request1\);)242 497 y(MPI_Intercomm_name_start\(m)o (yComm,)d("Connect)h(12",)i(&inter_request2\);)194 553 y(})194 666 y(/*)g(Everyone)f(has)g(the)h(same)f("FINISH")g(code...)g (*/)194 723 y(MPI_Intercomm_finish)e(\(inter_request1,)h (&myFirstComm\);)194 779 y(MPI_Intercomm_finish)f(\(inter_request2,)h (&mySecondComm\);)194 892 y(/*)i(Do)f(some)h(work)f(...)h(*/)194 1005 y(MPI_Finalize\(\);)147 1061 y(})1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 139 147 139 146 bop 75 358 a Fu(Chapter)34 b(6)75 568 y Fy(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)75 811 y Ft(6.1)59 b(Intro)r(duction)75 916 y FC(A)13 b(pro)q(cess)h(group)f(in)h(MPI)g(is)g(a)f(collection)i (of)e Fw(n)g FC(pro)q(cesses.)20 b(Eac)o(h)13 b(pro)q(cess)h(in)g(the)f (group)g(is)h(assigned)75 972 y(a)j(rank)g(b)q(et)o(w)o(een)g(0)g(and)g Fw(n)12 b FA(\000)f FC(1.)25 b(In)18 b(man)o(y)f(parallel)h (applications)h(a)e(linear)h(pro)q(cess)f(ordering)h(do)q(es)75 1029 y(not)c(re\015ect)h(the)g(comm)o(unication)g(pattern)f(of)g(the)h (pro)q(cesses,)f(whic)o(h)i(is)f(usually)h(determined)f(b)o(y)g(the)75 1085 y(underlying)20 b(problem)e(geometry)f(and)h(the)f(n)o(umerical)i (algorithm)f(used.)28 b(Often)18 b(the)f(pro)q(cesses)h(are)75 1142 y(arranged)13 b(in)g(top)q(ological)h(patterns)f(as)g(simple)h(as) f(a)g(t)o(w)o(o{)e(or)i(three{dimensional)i(grid.)k(In)14 b(the)f(more)75 1198 y(general)19 b(case)f(the)h(logical)g(pro)q(cess)g (arrangemen)o(t)e(is)i(describ)q(ed)h(b)o(y)e(a)g(graph.)29 b(In)19 b(this)g(c)o(hapter)f(w)o(e)75 1255 y(will)f(refer)e(to)f(this) i(logical)h(pro)q(cess)e(arrangemen)o(t)f(as)h(the)g(\\virtual)h(top)q (ology)l(.")166 1313 y(A)f(clear)h(distinction)h(m)o(ust)d(b)q(e)i (made)f(b)q(et)o(w)o(een)h(the)f(virtual)h(pro)q(cess)f(top)q(ology)g (and)g(the)h(top)q(ol-)75 1369 y(ogy)21 b(of)g(the)g(ph)o(ysical)i (hardw)o(are)e(pro)q(cessors.)38 b(The)21 b(application-sp)q(eci)q (\014c,)26 b(virtual)c(top)q(ology)f(can)75 1426 y(b)q(e)f(exploited)h (b)o(y)f(the)f(system)g(in)i(the)e(assignmen)o(t)h(of)f(pro)q(cesses)h (to)f(ph)o(ysical)i(pro)q(cessors,)f(if)g(this)75 1482 y(helps)14 b(to)f(impro)o(v)o(e)g(the)g(comm)o(unication)h(p)q (erformance)f(on)g(a)g(giv)o(en)h(mac)o(hine.)20 b(Ho)o(w)12 b(this)i(mapping)f(is)75 1539 y(done)i(in)h(detail,)f(ho)o(w)o(ev)o (er,)f(is)h(outside)g(the)g(scop)q(e)g(of)g(MPI.)f(The)h(description)h (of)f(the)f(virtual)i(pro)q(cess)75 1595 y(top)q(ology)l(,)g(on)h(the)f (other)g(hand,)h(only)g(dep)q(ends)h(on)e(the)h(application,)h(and)e (is,)h(therefore,)f(mac)o(hine{)75 1652 y(indep)q(enden)o(t.)22 b(The)13 b(functions)h(whic)o(h)g(are)f(prop)q(osed)h(in)g(this)g(pap)q (er)g(only)f(deal)h(with)g(this)g(side)g(of)f(the)75 1708 y(mapping)j(problem.)166 1767 y(There)e(are)g(w)o(ell{kno)o(wn)h (tec)o(hniques)g(for)f(mapping)h(grid/torus)e(structures)h(to)g(hardw)o (are)f(top)q(o-)75 1823 y(logies)j(suc)o(h)g(as)g(h)o(yp)q(ercub)q(es)h (or)e(grids.)22 b(F)l(or)15 b(more)g(complicated)i(graph)f(structures)f (go)q(o)q(d)g(heuristics)75 1879 y(often)j(yield)j(near{optimal)e (results)g([16)o(].)30 b(On)19 b(the)g(other)g(hand,)g(if)g(there)g(is) g(no)g(w)o(a)o(y)f(for)g(the)h(user)75 1936 y(to)f(sp)q(ecify)i(the)f (logical)h(pro)q(cess)f(arrangemen)o(t)f(as)g(a)g(\\virtual)h(top)q (ology)l(,")g(a)g(random)f(mapping)h(is)75 1992 y(most)f(lik)o(ely)i (the)e(consequence.)31 b(On)19 b(some)g(mac)o(hines)g(this)g(will)h (lead)f(to)f(unnecessary)h(con)o(ten)o(tion)75 2049 y(in)d(the)g(in)o (terconnect)h(net)o(w)o(ork.)j(Therefore,)15 b(though)h(the)f(ph)o (ysical)i(mapping)g(is)f(not)f(discussed,)i(the)75 2105 y(mere)g(existence)i(of)d(the)i(virtual)f(top)q(ology)g(information)g (ma)o(y)g(b)q(e)h(used)f(as)g(advice)h(b)o(y)f(the)h(run)o(time)75 2162 y(system.)41 b(Some)23 b(details)g(ab)q(out)f(predicted)i(and)f (measured)f(p)q(erformance)h(impro)o(v)o(emen)o(ts)f(whic)o(h)75 2218 y(result)14 b(from)e(go)q(o)q(d)h(pro)q(cess{to{pro)q(cessor)f (mapping)i(on)f(mo)q(dern)h(w)o(ormhole{routing)f(arc)o(hitectures)75 2275 y(can)i(b)q(e)h(found)g(in)g([8)o(,)f(7].)75 2429 y Ft(6.2)59 b(Virtual)20 b(T)-5 b(op)r(ologies)75 2534 y FC(In)13 b(the)g(most)f(general)h(case)g(the)g(comm)o(unication)g (pattern)g(of)f(a)g(set)h(of)f(pro)q(cesses)h(can)g(b)q(e)g(represen)o (ted)75 2591 y(b)o(y)g(a)g(graph.)19 b(The)14 b(no)q(des)f(stand)h(for) e(the)i(pro)q(cesses,)f(and)h(the)f(edges)h(connect)f(pro)q(cesses)h (whic)o(h)g(com-)75 2647 y(m)o(unicate)19 b(with)h(eac)o(h)f(other.)30 b(MPI)19 b(pro)o(vides)g(message{passing)g(b)q(et)o(w)o(een)g(an)o(y)g (pair)g(of)g(pro)q(cesses)75 2704 y(in)f(a)f(group.)25 b(There)17 b(is)h(no)f(requiremen)o(t)h(for)e(op)q(ening)i(a)f(c)o (hannel)h(explicitly)l(.)29 b(Therefore,)17 b(a)g(\\miss-)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 140 148 140 147 bop 75 -100 a FC(140)882 b Fs(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y FC(ing)k(link")g(in)g(the)f (user{de\014ned)h(pro)q(cess)g(graph)e(do)q(es)i(not)e(prev)o(en)o(t)h (the)h(corresp)q(onding)g(pro)q(cesses)75 102 y(from)e(exc)o(hanging)h (messages.)27 b(It)17 b(only)h(means)g(that)f(this)h(connection)g(is)g (neglected)h(in)g(the)e(virtual)75 158 y(top)q(ology)l(.)23 b(This)17 b(implies)i(that)c(the)i(top)q(ology)f(giv)o(es)g(no)h(con)o (v)o(enien)o(t)f(w)o(a)o(y)g(of)g(naming)h(this)f(path)o(w)o(a)o(y)75 214 y(of)d(comm)o(unication.)20 b(Another)13 b(p)q(ossible)i (consequence)g(is)f(that)f(an)g(automatic)g(mapping)h(to)q(ol)f(\(if)h (one)75 271 y(exists)h(for)e(the)i(run)o(time)f(en)o(vironmen)o(t\))g (will)i(not)e(try)g(to)f(assign)i(the)f(pro)q(cesses)h(to)e(neigh)o(b)q (oring)j(pro-)75 327 y(cessors.)i(Edges)12 b(in)h(the)e(comm)o (unication)i(graph)e(are)h(not)f(w)o(eigh)o(ted.)19 b(Pro)q(cesses)11 b(are)h(either)g(neigh)o(b)q(ors)75 384 y(or)i(not.)20 b(Exp)q(erience)d(with)e(similar)h(tec)o(hniques)g(in)g(P)l(ARMA)o(CS)g ([4)o(,)e(6])h(sho)o(w)f(that)g(this)i(information)75 440 y(is)i(usually)h(su\016cien)o(t)g(for)e(a)g(go)q(o)q(d)h(mapping.) 28 b(Additionally)l(,)20 b(a)e(more)f(precise)i(sp)q(eci\014cation)h(w) o(ould)75 497 y(b)q(e)g(more)f(di\016cult)i(for)d(the)i(user)f(to)g (set)g(up,)h(and)f(it)h(w)o(ould)g(mak)o(e)e(the)i(in)o(terface)f (functions)h(more)75 553 y(complicated.)166 610 y(In)c(principle,)h(a)e (function)h(for)f(the)g(de\014nition)i(of)e(suc)o(h)h(a)f(graph)g (structure)g(w)o(ould)g(b)q(e)h(su\016cien)o(t)75 667 y(for)d(all)i(applications.)21 b(Ho)o(w)o(ev)o(er,)13 b(in)h(man)o(y)g(applications)h(the)f(graph)g(structure)f(is)h(v)o(ery) g(regular,)g(and)75 723 y(the)e(detailed)i(set{up)f(of)e(the)i(graph)f (w)o(ould)h(b)q(e)g(incon)o(v)o(enien)o(t)g(for)f(the)g(user)h(and)f (resource{consuming)75 780 y(during)g(run)g(time.)19 b(A)12 b(large)f(fraction)g(of)g(all)i(parallel)g(applications)g(use)f (pro)q(cess)g(top)q(ologies)f(lik)o(e)i(rings,)75 836 y(t)o(w)o(o{)i(or)i(higher{dimensional)i(grids,)e(or)g(tori.)24 b(These)18 b(structures)e(are)h(completely)h(de\014ned)g(b)o(y)f(the)75 892 y(n)o(um)o(b)q(er)d(of)g(dimensions)i(and)e(the)g(n)o(um)o(b)q(ers) h(of)f(pro)q(cesses)g(in)h(eac)o(h)f(co)q(ordinate)h(direction.)21 b(Also,)14 b(the)75 949 y(mapping)h(of)e(grids)h(and)h(tori)e(is)i (generally)g(an)f(easier)g(problem)h(then)f(that)f(of)h(general)g (graphs.)19 b(Th)o(us,)75 1005 y(it)c(is)h(desirable)h(to)d(separate)h (out)g(this)h(case)f(explicitly)l(.)166 1062 y(Pro)q(cess)i(co)q (ordinates)g(in)h(a)e(Cartesian)h(structure)g(b)q(egin)h(their)f(n)o (um)o(b)q(ering)h(at)e(0.)25 b(Ro)o(w)16 b(ma)s(jor)75 1119 y(n)o(um)o(b)q(ering)e(is)g(used)f(for)g(the)g(pro)q(cesses)g(in)h (a)f(Cartesian)g(structure.)19 b(This)14 b(means)f(that,)f(for)g (example,)75 1175 y(the)17 b(relation)h(b)q(et)o(w)o(een)g(group)e (rank)h(and)h(co)q(ordinates)f(for)g(four)g(pro)q(cesses)g(in)h(a)f (\(2)11 b FA(\002)h FC(2\))k(grid)i(is)f(as)75 1232 y(follo)o(ws.)206 1314 y(co)q(ord)e(\(0,0\):)48 b(rank)15 b(0)206 1371 y(co)q(ord)g(\(0,1\):)48 b(rank)15 b(1)206 1427 y(co)q(ord)g(\(1,0\):) 48 b(rank)15 b(2)206 1484 y(co)q(ord)g(\(1,1\):)48 b(rank)15 b(3)75 1618 y Ft(6.3)59 b(Emb)r(edding)18 b(in)i(MPI)75 1720 y FC(The)13 b(supp)q(ort)f(for)g(virtual)h(top)q(ologies)g(as)f (de\014ned)h(in)h(this)e(c)o(hapter)h(is)f(consisten)o(t)h(with)g (other)f(parts)f(of)75 1777 y(MPI,)h(and,)h(whenev)o(er)g(p)q(ossible,) h(mak)o(es)e(use)h(of)f(functions)h(whic)o(h)h(are)e(de\014ned)i (elsewhere.)20 b(T)l(op)q(ology)75 1833 y(information)h(is)h(asso)q (ciated)g(with)g(comm)o(unicators.)38 b(It)21 b(can)g(b)q(e)h(main)o (tained)h(using)f(the)f(cac)o(hing)75 1890 y(mec)o(hanism)d(describ)q (ed)g(in)g(Chapter)f(5.)25 b(The)17 b(top)q(ology)f(functions)i (complemen)o(t)g(and)f(in)o(terop)q(erate)75 1946 y(with)f(the)f (collectiv)o(e)i(comm)o(unication)f(functions)g(describ)q(ed)h(in)f (Chapter)f(4.)166 2003 y(The)d(curren)o(t)g(design)g(of)g(the)f(pro)q (cess)h(top)q(ology)g(functions)g(assumes)g(that)f(MPI)h(is)g(free)g (to)f(c)o(ho)q(ose)75 2059 y(an)o(y)g(pro)q(cess)h(in)h(a)e(group)g (for)g(an)o(y)h(lo)q(cation)g(in)h(the)e(top)q(ology)l(.)19 b(There)12 b(are)f(no)h(sp)q(ecial)h(pro)q(cesses)f(whic)o(h)75 2116 y(ha)o(v)o(e)g(to)f(b)q(e)i(aligned)g(with)g(top)q(ology)e (substructures.)19 b(This)13 b(assumption)f(\014ts)g(b)q(est)g(to)g(a)g (\\lo)q(cal)g(SPMD)75 2172 y(mo)q(del")k(in)g(whic)o(h)g(all)g(pro)q (cesses)g(in)g(the)f(same)g(group)g(execute)h(the)f(same)g(program.)75 2318 y Ft(6.4)59 b(Overview)19 b(of)h(the)f(p)n(rop)r(osed)g(MPI)h (functions)75 2421 y FC(The)13 b(functions)h Fv(MPI)p 445 2421 14 2 v 16 w(MAKE)p 590 2421 V 16 w(GRAPH)g FC(and)f Fv(MPI)p 938 2421 V 16 w(MAKE)p 1083 2421 V 16 w(CART)g FC(are)g(used)g(to)f(create)h(general)g(virtual)75 2477 y(top)q(ologies)21 b(and)f(Cartesian)g(top)q(ologies,)h(resp)q(ectiv)o (ely)l(.)37 b(These)20 b(top)q(ology)g(creation)g(functions)h(are)75 2534 y(collectiv)o(e.)f(As)11 b(for)g(other)g(collectiv)o(e)i(calls,)f (the)f(co)q(de)h(should)g(b)q(e)g(written)f(to)g(w)o(ork)f(correctly)l (,)i(whether)75 2590 y(the)j(call)i(sync)o(hronizes)f(or)f(not.)166 2647 y(The)g(top)q(ology)g(de\014nition)i(function)g Fv(MPI)p 912 2647 V 15 w(MAKE)p 1056 2647 V 17 w(CART)e FC(can)h(b)q(e)g(used)g(to)e(describ)q(e)j(Cartesian)75 2704 y(structures)k(of)g(arbitrary)f(dimension.)40 b(F)l(or)21 b(eac)o(h)g(co)q(ordinate)h(direction)g(it)g(is)f(sp)q(eci\014ed)j (whether)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 141 149 141 148 bop 75 -100 a Fs(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FC(141)75 45 y(the)16 b(pro)q(cess)g(structure)g(is)g(p)q(erio)q(dic) i(or)d(not.)22 b(Similar)17 b(functions)g(are)e(con)o(tained)i(in)g (EXPRESS)f([17)o(])75 102 y(and)g(P)l(ARMA)o(CS.)h(Note)f(that)f(an)h Fw(n)p FC({dimensional)j(h)o(yp)q(ercub)q(e)f(is)e(an)h Fw(n)p FC({dimensional)h(torus)e(with)g(2)75 158 y(pro)q(cesses)i(p)q (er)h(co)q(ordinate)f(direction.)29 b(Th)o(us,)18 b(sp)q(ecial)i(supp)q (ort)e(for)f(h)o(yp)q(ercub)q(e)i(structures)f(is)g(not)75 214 y(necessary)l(.)h(The)12 b(lo)q(cal)h(auxiliary)g(function)g Fv(MPI)p 929 214 14 2 v 16 w(MAKE)p 1074 214 V 16 w(DIMS)e FC(can)h(b)q(e)h(used)f(to)f(compute)h(an)g(\\ev)o(en")75 271 y(distribution)17 b(of)e(pro)q(cesses)g(among)g(a)g(giv)o(en)g(n)o (um)o(b)q(er)h(of)f(dimensions.)166 329 y(The)j(function)g Fv(MPI)p 527 329 V 16 w(TOPO)p 669 329 V 18 w(ST)l(A)l(TUS)h FC(can)e(b)q(e)i(used)f(to)f(inquire)j(on)d(the)h(top)q(ology)g(asso)q (ciated)75 385 y(with)e(a)f(comm)o(unicator.)20 b(The)c(top)q(ological) g(information)f(can)h(b)q(e)g(extracted)f(from)g(the)g(comm)o(unica-)75 442 y(tor)h(using)h(the)g(functions)h Fv(MPI)p 633 442 V 15 w(GET)p 736 442 V 17 w(GRAPHDIMS)f FC(and)g Fv(MPI)p 1203 442 V 16 w(GET)p 1307 442 V 17 w(GRAPH)p FC(,)g(for)f(general)i (graphs,)75 498 y(and)c Fv(MPI)p 247 498 V 16 w(GET)p 351 498 V 16 w(CARTDIM)g FC(and)g Fv(MPI)p 755 498 V 16 w(GET)p 859 498 V 16 w(CART)p FC(,)g(for)f(Cartesian)h(top)q (ologies.)19 b(Sev)o(eral)c(additional)75 555 y(functions)f(are)e(pro)o (vided)i(to)e(manipulate)i(Cartesian)f(top)q(ologies:)19 b(the)13 b(functions)h Fv(MPI)p 1603 555 V 16 w(CART)p 1738 555 V 17 w(RANK)75 611 y FC(and)f Fv(MPI)p 246 611 V 15 w(CART)p 380 611 V 17 w(COORDS)h FC(translate)e(Cartesian)g(co)q (ordinates)h(in)o(to)f(a)g(group)g(rank,)g(and)h(vice-v)o(ersa;)75 668 y(the)g(function)h Fv(MPI)p 412 668 V 16 w(CART)p 547 668 V 17 w(SUB)g FC(can)f(b)q(e)h(used)f(to)g(extract)f(a)h (Cartesian)g(subspace;)h(and)f(the)g(function)75 724 y Fv(MPI)p 160 724 V 16 w(CART)p 295 724 V 17 w(SHIFT)g FC(pro)o(vides)h(the)f(information)h(needed)g(to)f(execute)h(a)f(shift) g(in)i(a)e(Cartesian)g(dimen-)75 781 y(sion.)20 b(The)c(t)o(w)o(o)e (functions)i Fv(MPI)p 647 781 V 16 w(GRAPH)p 813 781 V 17 w(NEIGHBORS)p 1081 781 V 18 w(COUNT)g FC(and)g Fv(MPI)p 1442 781 V 16 w(GRAPH)p 1608 781 V 17 w(NEIGHBORS)75 837 y FC(can)e(b)q(e)h(used)f(to)g(extract)f(the)h(neigh)o(b)q(ors)h (of)f(a)f(no)q(de)i(in)g(a)f(graph.)19 b(The)14 b(function)h Fv(MPI)p 1595 837 V 16 w(CART)p 1730 837 V 17 w(SUB)f FC(is)75 893 y(collectiv)o(e;)j(all)f(other)f(functions)h(are)e(lo)q (cal.)166 951 y(Tw)o(o)d(additional)i(functions,)g Fv(MPI)p 767 951 V 16 w(MAP)p 882 951 V 17 w(GRAPH)g FC(and)f Fv(MPI)p 1229 951 V 16 w(MAP)p 1344 951 V 16 w(CART)h FC(are)f(presen)o(ted)g(in)h(the)75 1008 y(last)19 b(section.)31 b(In)19 b(general)g(these)g(functions)h(will)g(not)e(b)q(e)i(called)g (b)o(y)f(the)f(user)h(directly)l(.)32 b(Ho)o(w)o(ev)o(er,)75 1064 y(together)16 b(with)i(the)f(comm)o(unicator)g(manipulation)h (functions)g(presen)o(ted)g(in)g(Chapter)e(5,)h(they)g(are)75 1121 y(su\016cien)o(t)23 b(to)e(implemen)o(t)j(all)f(other)e(top)q (ology)h(functions.)41 b(The)23 b(last)f(section)g(outlines)i(suc)o(h)e (an)75 1177 y(implemen)o(tation.)75 1329 y Ft(6.5)59 b(T)-5 b(op)r(ology)19 b(Constructo)n(rs)75 1481 y Fv(MPI)p 160 1481 V 16 w(MAKE)p 305 1481 V 16 w(CART\(comm)p 575 1481 V 15 w(old,)c(ndims,)g(dims,)f(p)q(erio)q(ds,)i(reo)o(rder,)e (comm)p 1370 1481 V 15 w(ca)o(rt\))117 1559 y Fq(IN)171 b Fv(comm)p 454 1559 V 14 w(old)383 b Fq(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g(\(handle\))117 1638 y(IN)171 b Fv(ndims)460 b Fq(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i (Cartesian)g(grid)f(\(in)o(teger\))117 1716 y(IN)171 b Fv(dims)483 b Fq(in)o(teger)19 b(arra)o(y)f(of)f(size)i Fl(ndims)f Fq(sp)q(ecifying)g(the)g(n)o(um)o(b)q(er)g(of)905 1772 y(pro)q(cesses)f(in)c(eac)o(h)i(dimension)117 1850 y(IN)171 b Fv(p)q(erio)q(ds)436 b Fq(logical)12 b(arra)o(y)h(of)g(size) h Fl(ndims)f Fq(sp)q(ecifying)g(whether)h(the)g(grid)905 1907 y(is)d(p)q(erio)q(dic)g(\()p Fl(true)p Fq(\))h(or)f(ap)q(erio)q (dic)g(\()p Fl(false)p Fq(\))g(in)f(eac)o(h)i(dimension)117 1985 y(IN)171 b Fv(reo)o(rder)438 b Fq(ranking)10 b(ma)o(y)g(b)q(e)h (reordered)i(\()p Fl(true)p Fq(\))f(or)f(not)g(\()p Fl(false)p Fq(\))g(\(logical\))117 2063 y(OUT)124 b Fv(comm)p 454 2063 V 14 w(ca)o(rt)367 b Fq(comm)o(unicator)11 b(for)j(new)g (Cartesian)g(group)g(\(handle\))75 2189 y Fo(int)23 b(MPI)p 245 2189 15 2 v 17 w(Make)p 358 2189 V 17 w(cart\(MPI)p 567 2189 V 16 w(comm)g(comm)p 798 2189 V 17 w(old,)g(int)h(ndims,)f (int)g(*dims,)g(int)h(*periods,)393 2245 y(int)g(reorder,)e(MPI)p 778 2245 V 17 w(comm)h(*comm)p 1034 2245 V 17 w(cart\))75 2333 y(MPI)p 150 2333 V 17 w(MAKE)p 263 2333 V 16 w(CART\(COMM)p 495 2333 V 16 w(OLD,)h(NDIMS,)f(DIMS,)g(PERIODS,)g(REORDER,)f(COMM)p 1466 2333 V 17 w(CART,)h(IERROR\))170 2390 y(INTEGER)g(COMM)p 460 2390 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(PERIODS\(*\),)f(REORDER,)h (COMM)p 1575 2390 V 17 w(CART,)170 2446 y(IERROR)166 2534 y FC(The)13 b(function)h(returns)f(a)g(handle)h(to)e(a)h(new)g (comm)o(unicator)g(to)f(whic)o(h)i(the)g(Cartesian)e(top)q(ology)75 2591 y(information)i(is)g(attac)o(hed.)19 b(If)14 b Fv(reo)o(rder)f(=)h (false)h FC(then)f(the)g(rank)f(of)h(eac)o(h)g(pro)q(cess)g(in)h(the)f (new)g(group)f(is)75 2647 y(iden)o(tical)19 b(to)d(its)h(rank)g(in)g (the)g(old)h(group.)25 b(Otherwise,)17 b(the)g(function)h(ma)o(y)e (reorder)h(the)g(pro)q(cesses,)75 2704 y(so)i(as)f(to)g(c)o(ho)q(ose)h (a)g(go)q(o)q(d)g(em)o(b)q(edding)i(of)d(the)h(virtual)h(top)q(ology)e (on)o(to)g(the)h(ph)o(ysical)i(mac)o(hine.)32 b(If)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 142 150 142 149 bop 75 -100 a FC(142)882 b Fs(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y FC(the)j(total)f(size)h(of)g(the)f (Cartesian)h(grid)g(is)g(smaller)h(than)e(the)h(size)h(of)e(the)h (group)f(of)g Fv(comm)p FC(,)f(then)i(a)75 102 y(pro)q(cess)e(whic)o(h) h(is)g(not)e(a)h(mem)o(b)q(er)g(of)f(the)i(grid)f(is)g(returned)h(a)e (n)o(ull)j(handle.)23 b(The)17 b(call)g(is)f(erroneous)75 158 y(if)g(it)f(sp)q(eci\014es)i(a)e(grid)h(that)e(is)i(larger)f(than)g (the)g(group)g(size.)166 237 y(In)22 b(a)g(p)q(ortable)g(program)f(the) h(n)o(um)o(b)q(er)g(of)f(a)o(v)m(ailable)j(pro)q(cessors)d(is)h (usually)i(not)d(kno)o(wn)h(at)75 294 y(compile)g(time.)36 b(In)21 b(the)f(case)h(of)f(a)g(Cartesian)g(top)q(ology)g(the)h (function)g Fv(MPI)p 1485 294 14 2 v 16 w(MAKE)p 1630 294 V 16 w(DIMS)f FC(helps)75 350 y(in)e(setting)e(the)h(n)o(um)o(b)q (ers)g(of)f(pro)q(cesses)i(p)q(er)f(co)q(ordinate)g(direction,)h(dep)q (ending)h(on)e(the)f(n)o(um)o(b)q(er)h(of)75 407 y(pro)q(cesses)f(in)g (the)f(group)g(and)g(optional)h(constrain)o(ts)f(whic)o(h)h(can)f(b)q (e)h(sp)q(eci\014ed)h(b)o(y)e(the)h(user:)75 533 y Fv(MPI)p 160 533 V 16 w(MAKE)p 305 533 V 16 w(DIMS\(nno)q(des,)g(ndims,)f (dims\))117 633 y Fq(IN)171 b Fv(nno)q(des)441 b Fq(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(a)f(grid)h(\(in)o(teger\))117 754 y(IN)171 b Fv(ndims)460 b Fq(n)o(um)o(b)q(er)13 b(of)h(Cartesian)g (dimensions)e(\(in)o(teger\))117 875 y(INOUT)78 b Fv(dims)483 b Fq(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fk(ndims)e Fq(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 931 y(no)q(des)e(in)e(eac)o(h)i(dimension)75 1079 y Fo(int)23 b(MPI)p 245 1079 15 2 v 17 w(Make)p 358 1079 V 17 w(dims\(int)g (nnodes,)g(int)g(ndims,)g(int)g(*dims\))75 1188 y(MPI)p 150 1188 V 17 w(MAKE)p 263 1188 V 16 w(DIMS\(NNODES,)g(NDIMS,)g(DIMS,)g (IERROR\))170 1244 y(INTEGER)g(NNODES,)g(NDIMS,)g(DIMS\(*\),)g(IERROR) 166 1354 y FC(The)12 b(en)o(tries)h(in)g(the)f(arra)o(y)f Fv(dims)h FC(are)g(set)g(to)f(describ)q(e)j(a)e(Cartesian)g(grid)g (with)h Fv(ndims)f FC(dimensions)75 1410 y(and)h(a)g(total)g(of)f Fv(nno)q(des)k FC(no)q(des.)j(The)14 b(dimensions)h(are)d(set)h(to)g(b) q(e)h(as)e(close)i(to)f(eac)o(h)g(other)g(as)g(p)q(ossible.)75 1466 y(The)20 b(caller)h(ma)o(y)e(further)h(constrain)g(the)g(op)q (eration)g(of)g(this)g(routine)g(b)o(y)g(sp)q(ecifying)i(elemen)o(ts)e (of)75 1523 y(arra)o(y)13 b Fo(dims)p FC(.)19 b(If)14 b Fo(dims[i])g FC(is)g(set)g(to)g(a)g(p)q(ositiv)o(e)h(n)o(um)o(b)q (er,)g(the)f(routine)h(will)h(not)d(mo)q(dify)i(the)g(n)o(um)o(b)q(er) 75 1579 y(of)20 b(no)q(des)h(in)g(the)f(corresp)q(onding)h(dimension;)k (only)20 b(those)g(en)o(tries)h(where)g Fo(dims[i])i(=)g(0)d FC(will)i(b)q(e)75 1636 y(mo)q(di\014ed)17 b(b)o(y)e(the)g(call.)166 1715 y(Note)g(that)f(an)h(error)g(will)i(o)q(ccur)e(if)h Fo(nnodes)e FC(is)i(not)f(a)g(m)o(ultiple)i(of)1413 1675 y Fg(Y)1352 1769 y Fh(i;dims)p Fi([)p Fh(i)p Fi(])p Fj(6)p Fi(=0)1535 1715 y Fw(dims)p FC([)p Fw(i)p FC(].)166 1840 y(Negativ)o(e)f(input)i(v)m(alues)f(of)f Fo(dims[i])g FC(will)i(also)e(cause)h(an)f(error.)23 b(All)18 b(elemen)o(ts)f(of)f Fo(dims)g FC(m)o(ust)75 1896 y(b)q(e)g(set)f(to)f(0)h(for)g(completely) h(automatic)f(pro)q(cess)g(distribution.)166 1975 y(F)l(or)h Fo(dims[i])h FC(set)f(b)o(y)h(the)h(call,)g Fo(dims[i])e FC(will)j(b)q(e)f(ordered)f(in)h(non{increasing)g(order.)26 b(Arra)o(y)75 2032 y Fo(dims)10 b FC(is)g(suitable)i(for)d(use)i(as)e (input)j(to)d(routine)i Fv(MPI)p 978 2032 14 2 v 15 w(MAKE)p 1122 2032 V 17 w(CART)p FC(.)f(The)g(function)h Fv(MPI)p 1624 2032 V 16 w(MAKE)p 1769 2032 V 16 w(DIMS)75 2088 y FC(is)16 b(lo)q(cal.)166 2187 y(Examples:)p 166 2225 1206 2 v 165 2281 2 57 v 191 2264 a Fo(dims)p 420 2281 V 158 w FC(function)g(call)p 1047 2281 V 383 w Fo(dims)p 1370 2281 V 165 2338 V 191 2321 a FC(b)q(efore)f(call)p 420 2338 V 1047 2338 V 679 w(on)g(return)p 1370 2338 V 166 2339 1206 2 v 165 2396 2 57 v 191 2379 a(\(0,0\))p 420 2396 V 159 w Fv(MPI)p 530 2379 14 2 v 16 w(MAKE)p 675 2379 V 16 w(DIMS\(6,)f(2,)g(dims\))p 1047 2396 2 57 v 50 w FC(\(3,2\))p 1370 2396 V 165 2452 V 191 2435 a(\(0,0\))p 420 2452 V 159 w Fv(MPI)p 530 2435 14 2 v 16 w(MAKE)p 675 2435 V 16 w(DIMS\(7,)g(2,)g(dims\))p 1047 2452 2 57 v 50 w FC(\(7,1\))p 1370 2452 V 165 2509 V 191 2492 a(\(0,3,0\))p 420 2509 V 123 w Fv(MPI)p 530 2492 14 2 v 16 w(MAKE)p 675 2492 V 16 w(DIMS\(6,)g(3,)g(dims\))p 1047 2509 2 57 v 50 w FC(\(2,3,1\))p 1370 2509 V 165 2565 V 191 2548 a(\(0,3,0\))p 420 2565 V 123 w Fv(MPI)p 530 2548 14 2 v 16 w(MAKE)p 675 2548 V 16 w(DIMS\(7,)g(3,)g(dims\))p 1047 2565 2 57 v 50 w FC(erroneous)h(call)p 1370 2565 V 166 2567 1206 2 v 166 2704 a(F)l(or)f(a)h(general)h(graph)f (structure,)g(the)g(constructor)f(is)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 143 151 143 150 bop 75 -100 a Fs(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FC(143)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(MAKE)p 305 45 V 16 w(GRAPH\(comm)p 606 45 V 15 w(old,)15 b(nno)q(des,)i(index,)f (edges,)g(reo)o(rder,)d(comm)p 1401 45 V 15 w(graph\))117 126 y Fq(IN)171 b Fv(comm)p 454 126 V 14 w(old)383 b Fq(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g(\(handle\))117 208 y(IN)171 b Fv(nno)q(des)441 b Fq(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(graph)g(\(in)o(teger\))117 290 y(IN)171 b Fv(index)473 b Fq(arra)o(y)14 b(of)f(in)o(tegers)i (describing)f(no)q(de)h(degrees)g(\(see)g(b)q(elo)o(w\))117 372 y(IN)171 b Fv(edges)468 b Fq(arra)o(y)14 b(of)f(in)o(tegers)i (describing)f(graph)g(edges)h(\(see)g(b)q(elo)o(w\))117 454 y(IN)171 b Fv(reo)o(rder)438 b Fq(ranking)10 b(ma)o(y)g(b)q(e)h (reordered)i(\()p Fl(true)p Fq(\))f(or)f(not)g(\()p Fl(false)p Fq(\))g(\(logical\))117 536 y(OUT)124 b Fv(comm)p 454 536 V 14 w(graph)333 b Fq(comm)o(unicator)11 b(for)j(new)g(graph)g (group)g(\(handle\))75 664 y Fo(int)23 b(MPI)p 245 664 15 2 v 17 w(Make)p 358 664 V 17 w(graph\(MPI)p 591 664 V 16 w(comm)g(comm)p 822 664 V 17 w(old,)g(int)h(nnodes,)e(int)i (*index,)f(int)g(*edges,)393 720 y(int)h(reorder,)e(MPI)p 778 720 V 17 w(comm)h(*comm)p 1034 720 V 17 w(graph\))75 810 y(MPI)p 150 810 V 17 w(MAKE)p 263 810 V 16 w(GRAPH\(COMM)p 519 810 V 16 w(OLD,)h(NNODES,)e(INDEX,)h(EDGES,)h(REORDER,)e(COMM)p 1490 810 V 17 w(GRAPH,)h(IERROR\))170 866 y(INTEGER)g(COMM)p 460 866 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g(EDGES\(*\),)f(REORDER,)h (COMM)p 1575 866 V 17 w(GRAPH,)170 923 y(IERROR)166 1013 y FC(The)c(function)h(returns)f(a)f(handle)i(to)e(a)h(new)g(comm)o (unicator)g(to)f(whic)o(h)i(the)f(graph)f(top)q(ology)75 1069 y(information)c(is)g(attac)o(hed.)19 b(If)14 b Fv(reo)o(rder)f(=)h (false)h FC(then)f(the)g(rank)f(of)h(eac)o(h)g(pro)q(cess)g(in)h(the)f (new)g(group)f(is)75 1126 y(iden)o(tical)19 b(to)d(its)h(rank)g(in)g (the)g(old)h(group.)25 b(Otherwise,)17 b(the)g(function)h(ma)o(y)e (reorder)h(the)g(pro)q(cesses,)75 1182 y(so)d(as)g(to)f(c)o(ho)q(ose)i (a)f(go)q(o)q(d)g(em)o(b)q(edding)i(of)d(the)i(virtual)g(top)q(ology)f (on)o(to)f(the)h(ph)o(ysical)i(mac)o(hine.)k(If)15 b(the)75 1239 y(size)f Fv(nno)q(des)i FC(of)c(the)i(graph)f(is)h(smaller)g(than) f(the)g(size)h(of)f(the)h(group)f(of)g Fv(comm)p FC(,)e(then)j(a)f(pro) q(cess)g(whic)o(h)75 1295 y(is)j(not)f(a)g(mem)o(b)q(er)g(of)g(the)g (grid)h(is)g(returned)g(a)f(n)o(ull)h(handle.)22 b(The)15 b(call)i(is)f(erroneous)f(if)h(it)f(sp)q(eci\014es)i(a)75 1351 y(graph)e(that)f(is)i(larger)f(than)g(the)g(group)g(size.)166 1411 y(The)e(three)h(parameters)e Fv(nno)q(des,)j(index)f FC(and)g Fv(edges)g FC(de\014ne)g(the)f(graph)g(structure.)19 b Fv(nno)q(des)c FC(is)f(the)75 1468 y(n)o(um)o(b)q(er)i(of)e(no)q(des) i(of)f(the)g(graph.)20 b(The)15 b(no)q(des)h(are)f(n)o(um)o(b)q(ered)h (from)e Fo(0)h FC(to)g Fo(nnodes-1)p FC(.)166 1528 y(The)g Fo(i)p FC(-th)g(en)o(try)g(of)g(arra)o(y)f Fv(index)i FC(stores)e(the)i(total)e(n)o(um)o(b)q(er)i(of)f(neigh)o(b)q(ors)g(of)g (the)g(\014rst)g Fo(i)g FC(graph)75 1584 y(no)q(des.)40 b(The)21 b(lists)i(of)e(neigh)o(b)q(ors)h(of)f(no)q(des)h Fo(0,)i(1,)g Fw(:)8 b(:)g(:)d Fo(,)24 b(nnodes-1)c FC(are)i(stored)f (in)h(consecutiv)o(e)75 1641 y(lo)q(cations)16 b(in)g(arra)o(y)e Fv(edges)p FC(.)166 1700 y(The)19 b(de\014nitions)h(of)e(the)h(argumen) o(ts)f Fo(nnodes)p FC(,)g Fo(index)p FC(,)h(and)g Fo(edges)f FC(are)g(illustrated)i(with)f(the)75 1757 y(follo)o(wing)d(simple)h (example:)166 1817 y(Assume)e(there)h(are)f(four)f(pro)q(cesses)i(0,)e (1,)h(2,)f(3)h(with)h(the)f(follo)o(wing)h(adjacency)g(matrix:)p 181 1871 433 2 v 180 1927 2 57 v 206 1910 a(pro)q(cess)p 373 1927 V 50 w(neigh)o(b)q(ors)p 613 1927 V 181 1929 433 2 v 180 1985 2 57 v 266 1968 a(0)p 373 1985 V 110 w(1,)f(3)p 613 1985 V 180 2042 V 266 2025 a(1)p 373 2042 V 110 w(0)p 613 2042 V 180 2098 V 266 2081 a(2)p 373 2098 V 110 w(3)p 613 2098 V 180 2154 V 266 2138 a(3)p 373 2154 V 110 w(0,)g(2)p 613 2154 V 181 2156 433 2 v 166 2234 a(Then,)g(the)h(input)g(argumen)o(ts)e(are:)206 2326 y(nno)q(des)i(=)50 b(4)206 2383 y(index)16 b(=)81 b(2,)15 b(3,)f(4,)h(6)206 2439 y(edges)g(=)82 b(1,)15 b(3,)f(0,)h(3,)f(0,)h(2)166 2534 y(Th)o(us,)h(in)h(C,)e Fo(index[0])g FC(is)i(the)f(degree)g(of)g(no)q(de)h(zero,)f(and)g Fo(index[i])23 b(-)h(index[i-1])14 b FC(is)j(the)75 2591 y(degree)22 b(of)g(no)q(de)g Fo(i,)i(i=1,)f Fw(:)8 b(:)g(:)e Fo(,)23 b(nnodes-1)p FC(;)h(the)e(list)h(of)e(neigh)o(b)q(ors)i(of)e (no)q(de)i(zero)e(is)i(stored)e(in)75 2647 y Fo(edges[j])p FC(,)d Fo(0)g FA(\024)g Fo(j)g FA(\024)g Fo(index)p FC([)p Fo(0)p FC(])11 b FA(\000)h Fo(1)18 b FC(and)h(the)g(list)g(of)f(neigh)o (b)q(ors)h(of)f(no)q(de)h Fo(i)p FC(,)f Fo(i)g Fw(>)h Fo(0)p FC(,)f(is)h(stored)f(in)75 2704 y Fo(edges[j])p FC(,)c Fo(index)p FC([)p Fo(i)9 b FA(\000)h Fo(1)p FC(])i FA(\024)h Fo(j)g FA(\024)g Fo(index)p FC([)p Fo(i)p FC(])8 b FA(\000)j Fo(1)p FC(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 144 152 144 151 bop 75 -100 a FC(144)882 b Fs(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)166 45 y FC(In)i(F)l(ortran,)f Fo(index[1])f FC(is)j(the)e(degree)i(of)e(no)q(de)h(zero,)g(and)g Fo(index[i+1])22 b(-)i(index[i])16 b FC(is)h(the)75 102 y(degree)22 b(of)g(no)q(de)g Fo(i,)i(i=1,)f Fw(:)8 b(:)g(:)e Fo(,)23 b(nnodes-1)p FC(;)h(the)e(list)h(of)e(neigh)o(b)q(ors)i(of)e (no)q(de)i(zero)e(is)i(stored)e(in)75 158 y Fo(edges[j])p FC(,)h Fo(1)i FA(\024)g Fo(j)g FA(\024)g Fo(index)p FC([)p Fo(1)p FC(])c(and)i(the)g(list)h(of)f(neigh)o(b)q(ors)g(of)g(no)q(de)g Fo(i)p FC(,)h Fo(i)h Fw(>)g Fo(0)p FC(,)f(is)g(stored)e(in)75 214 y Fo(edges[j])p FC(,)14 b Fo(index)p FC([)p Fo(i)p FC(])8 b(+)j Fo(1)h FA(\024)h Fo(j)g FA(\024)g Fo(index)p FC([)p Fo(i)c FC(+)h Fo(1)p FC(].)166 272 y(The)k(total)f(n)o(um)o(b)q (er)h(of)g(en)o(tries)g(in)h Fv(index)f FC(is)h Fo(nnodes)e FC(and)h(the)f(total)h(n)o(um)o(b)q(er)g(of)f(en)o(tries)h(in)h Fv(edges)75 329 y FC(is)h(equal)g(to)e(the)h(n)o(um)o(b)q(er)h(of)f (graph)g(edges.)166 462 y Fn(Implemen)o(tati)o(on)i(note:)54 b Fq(The)19 b(follo)o(wing)c(top)q(ology)i(information)f(is)i(lik)o (ely)f(to)h(b)q(e)h(stored)g(with)f(a)75 512 y(comm)o(unicator:)134 591 y(1.)k(T)o(yp)q(e)14 b(of)f(top)q(ology)g(\(Cartesian/graph\))134 670 y(2.)22 b(F)m(or)13 b(a)h(Cartesian)g(top)q(ology:)245 756 y Fc(\017)23 b Fq(ndims)12 b(\(n)o(um)o(b)q(er)h(of)h (dimensions\),)245 825 y Fc(\017)23 b Fq(dims)12 b(\(n)o(um)o(b)q(ers)i (of)f(pro)q(cesses)k(p)q(er)d(co)q(ordinate)h(direction\),)245 894 y Fc(\017)23 b Fq(p)q(erio)q(ds)14 b(\(p)q(erio)q(dicit)o(y)g (information\),)245 963 y Fc(\017)23 b Fq(o)o(wn)p 364 963 13 2 v 14 w(p)q(osition)13 b(\(o)o(wn)h(p)q(osition)f(in)h(grid,)f (could)g(also)h(b)q(e)g(computed)g(from)e(rank)h(and)h(dims\))134 1049 y(3.)22 b(F)m(or)13 b(a)h(graph)f(top)q(ology:)245 1135 y Fc(\017)23 b Fq(index,)13 b(edges)i(\(v)o(ectors)g(de\014ning)f (graph)g(structure\))166 1228 y(F)m(or)h(a)h(graph)f(structure)j(the)e (n)o(um)o(b)q(er)f(of)g(no)q(des)i(is)e(equal)h(to)f(the)i(n)o(um)o(b)q (er)e(of)g(pro)q(cesses)j(in)d(the)i(group.)75 1284 y(Therefore,)h(it)e (do)q(es)h(not)f(ha)o(v)o(e)g(to)g(b)q(e)h(stored)h(explicitly)m(.)23 b(An)17 b(additional)d(zero)k(en)o(try)e(at)h(the)g(start)g(of)e(arra)o (y)75 1341 y Fl(index)g Fq(will)e(simplify)e(access)k(to)f(the)h(top)q (ology)d(information.)75 1552 y Fp(6.5.1)49 b(T)l(op)q(ology)19 b(inquiry)e(functions)75 1640 y FC(If)h(a)f(top)q(ology)g(has)h(b)q (een)h(de\014ned)g(with)e(one)h(of)f(the)h(ab)q(o)o(v)o(e)f(functions,) i(the)f(top)q(ology)f(information)75 1697 y(can)g(b)q(e)g(lo)q(ok)o(ed) g(up)g(using)g(inquiry)h(functions.)24 b(They)17 b(all)h(are)e(lo)q (cal)h(calls,)h(and)e(do)h(not)f(require)h(an)o(y)75 1753 y(comm)o(unication.)166 1811 y(The)f(function)g Fv(MPI)p 523 1811 14 2 v 16 w(TOPO)p 665 1811 V 17 w(ST)l(A)l(TUS)h FC(returns)f(the)f(t)o(yp)q(e)h(of)f(top)q(ology)h(whic)o(h)g(is)g (assigned)g(to)f(a)75 1867 y(comm)o(unicator.)75 1972 y Fv(MPI)p 160 1972 V 16 w(TOPO)p 302 1972 V 17 w(ST)l(A)l(TUS\(comm,)f (status\))117 2051 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)11 b(\(handle\))117 2129 y(OUT)124 b Fv(status)460 b Fq(top)q(ology)13 b(t)o(yp)q(e)h(of)g(comm)o(uni)o(cator)e Fk(comm)h Fq(\(c)o(hoice\))75 2254 y Fo(int)23 b(MPI)p 245 2254 15 2 v 17 w(Topo)p 358 2254 V 17 w(status\(MPI)p 615 2254 V 16 w(comm)g(comm,)g(int)h (*status\))75 2342 y(MPI)p 150 2342 V 17 w(TOPO)p 263 2342 V 16 w(STATUS\(COMM,)f(STATUS,)g(IERROR\))170 2398 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)166 2486 y FC(The)15 b(output)g(v)m(alue)i Fo(status)d FC(is)i(one)f(of)g(the)g(follo)o (wing:)117 2590 y Fl(MPI)p 194 2590 13 2 v 14 w(GRAPH)602 b FC(graph)15 b(top)q(ology)117 2647 y Fl(MPI)p 194 2647 V 14 w(CART)631 b FC(Cartesian)15 b(top)q(ology)117 2704 y Fl(MPI)p 194 2704 V 14 w(UNDEFINED)508 b FC(no)15 b(top)q(ology)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 145 153 145 152 bop 75 -100 a Fs(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FC(145)166 45 y(F)l(unctions)19 b Fv(MPI)p 462 45 14 2 v 16 w(GET)p 566 45 V 16 w(GRAPHDIMS)g FC(and)g Fv(MPI)p 1036 45 V 16 w(GET)p 1140 45 V 16 w(GRAPH)h FC(retriev)o(e)e(the)g (graph)g(top)q(ology)75 102 y(information)d(that)g(w)o(as)f(asso)q (ciated)i(with)f(a)g(comm)o(unicator)g(b)o(y)g Fv(MPI)p 1310 102 V 16 w(MAKE)p 1455 102 V 16 w(GRAPH)p FC(.)75 209 y Fv(MPI)p 160 209 V 16 w(GET)p 264 209 V 17 w(GRAPHDIMS\(comm,)e (nno)q(des,)k(nedges\))117 290 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)7 b(for)i(group)h(with)f(graph)h(structure)i (\(handle\))117 373 y(OUT)124 b Fv(nno)q(des)441 b Fq(n)o(um)o(b)q(er) 13 b(of)h(no)q(des)g(in)g(graph)g(\(in)o(teger\))117 456 y(OUT)124 b Fv(nedges)445 b Fq(n)o(um)o(b)q(er)13 b(of)h(edges)h(in)e(graph)h(\(in)o(teger\))75 584 y Fo(int)23 b(MPI)p 245 584 15 2 v 17 w(Get)p 334 584 V 17 w(graphdims\(MPI)p 663 584 V 15 w(comm)h(comm,)f(int)g(*nnodes,)g(int)h(*nedges\))75 674 y(MPI)p 150 674 V 17 w(GET)p 239 674 V 17 w(GRAPHDIMS\(COMM,)d (NNODES,)i(NEDGES,)g(IERROR\))170 731 y(INTEGER)g(COMM,)h(NNODES,)e (NEDGES,)h(IERROR)166 821 y FC(The)e(information)g(pro)o(vided)h(b)o(y) f Fv(MPI)p 864 821 14 2 v 16 w(GET)p 968 821 V 16 w(GRAPHDIMS)h FC(can)f(b)q(e)g(used)h(to)e(dimension)j(the)75 877 y(v)o(ectors)14 b Fo(index)h FC(and)g Fo(edges)g FC(long)g(enough)h(in)g(the)f(follo)o (wing)h(call)g(to)f Fv(MPI)p 1389 877 V 16 w(GET)p 1493 877 V 16 w(GRAPH)p FC(.)75 985 y Fv(MPI)p 160 985 V 16 w(GET)p 264 985 V 17 w(GRAPH\(comm,)e(maxindex,)i(maxedges,)g(index,)h (edges\))117 1066 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)7 b(for)i(group)h(with)f(graph)h(structure)i(\(handle\))117 1149 y(IN)171 b Fv(maxindex)393 b Fq(length)14 b(of)f(v)o(ector)i Fl(index)g Fq(in)f(the)g(calling)f(program)905 1205 y(\(in)o(teger\)) 117 1288 y(IN)171 b Fv(maxedges)388 b Fq(length)14 b(of)f(v)o(ector)i Fl(edges)g Fq(in)e(the)i(calling)d(program)905 1344 y(\(in)o(teger\)) 117 1427 y(OUT)124 b Fv(index)473 b Fq(arra)o(y)17 b(of)g(in)o(tegers)i (con)o(taining)d(the)i(graph)g(structure)h(\(for)905 1483 y(details)14 b(see)h(the)g(de\014nition)e(of)g Fl(MPI)p 1484 1483 13 2 v 15 w(MAKE)p 1617 1483 V 14 w(GRAPH)p Fq(\))117 1566 y(OUT)124 b Fv(edges)468 b Fq(arra)o(y)14 b(of)f(in)o(tegers)i(con)o(taining)e(the)h(graph)g(structure)75 1695 y Fo(int)23 b(MPI)p 245 1695 15 2 v 17 w(Get)p 334 1695 V 17 w(graph\(MPI)p 567 1695 V 16 w(comm)g(comm,)g(int)h (maxindex,)f(int)g(maxedges,)g(int)g(*index,)393 1751 y(int)h(*edges\))75 1841 y(MPI)p 150 1841 V 17 w(GET)p 239 1841 V 17 w(GRAPH\(COMM,)e(MAXINDEX,)h(MAXEDGES,)f(INDEX,)h(EDGES,) g(IERROR\))170 1898 y(INTEGER)g(COMM,)h(MAXINDEX,)e(MAXEDGES,)h (INDEX\(*\),)f(EDGES\(*\),)h(IERROR)166 1988 y FC(The)13 b(functions)g Fv(MPI)p 535 1988 14 2 v 15 w(GET)p 638 1988 V 17 w(CARTDIM)f FC(and)h Fv(MPI)p 1040 1988 V 16 w(GET)p 1144 1988 V 16 w(CART)g FC(return)g(the)f(Cartesian)g(top)q (ology)75 2044 y(information)j(that)g(w)o(as)f(asso)q(ciated)i(with)f (a)g(comm)o(unicator)g(b)o(y)g(the)g(function)h Fv(MPI)p 1566 2044 V 16 w(MAKE)p 1711 2044 V 16 w(CART)p FC(.)75 2152 y Fv(MPI)p 160 2152 V 16 w(GET)p 264 2152 V 17 w(CARTDIM\(comm,)c (ndims\))117 2233 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j(\(han-)905 2289 y(dle\))117 2372 y(OUT)124 b Fv(ndims)460 b Fq(n)o(um)o(b)q(er)15 b(of)g(dimensions)f(of)g(the)i(Cartesian)g(structure)h(\(in-)905 2429 y(teger\))75 2557 y Fo(int)23 b(MPI)p 245 2557 15 2 v 17 w(Get)p 334 2557 V 17 w(cartdim\(MPI)p 615 2557 V 16 w(comm)g(comm,)g(int)h(*ndims\))75 2647 y(MPI)p 150 2647 V 17 w(GET)p 239 2647 V 17 w(CARTDIM\(COMM,)e(NDIMS,)h (IERROR\))170 2704 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 146 154 146 153 bop 75 -100 a FC(146)882 b Fs(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GET)p 264 45 V 17 w(CART\(comm,)e(maxdims,)g(dims,)i(p)q (erio)q(ds,)h(co)q(o)o(rds\))117 143 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j (\(han-)905 199 y(dle\))117 315 y(IN)171 b Fv(maxdims)403 b Fq(length)16 b(of)f(v)o(ectors)i Fk(dims,)k(periods)p Fq(,)15 b(and)g Fk(coords)g Fq(in)g(the)905 372 y(calling)e(program)f (\(in)o(teger\))117 488 y(OUT)124 b Fv(dims)483 b Fq(n)o(um)o(b)q(er)11 b(of)g(pro)q(cesses)k(for)c(eac)o(h)h(Cartesian)g(dimension)e(\(ar-)905 545 y(ra)o(y)k(of)f(in)o(teger\))117 661 y(OUT)124 b Fv(p)q(erio)q(ds)436 b Fq(p)q(erio)q(dicit)o(y)18 b(\()p Fl(true)p Fq(/)p Fl(false)p Fq(\))i(for)e(eac)o(h)h(Cartesian)g (dimension)905 717 y(\(arra)o(y)14 b(of)f(logical\))117 833 y(OUT)124 b Fv(co)q(o)o(rds)449 b Fq(co)q(ordinates)19 b(of)f(calling)f(pro)q(cess)j(in)d(Cartesian)i(structure)905 890 y(\(arra)o(y)14 b(of)f(in)o(teger\))75 1035 y Fo(int)23 b(MPI)p 245 1035 15 2 v 17 w(Get)p 334 1035 V 17 w(cart\(MPI)p 543 1035 V 16 w(comm)g(comm,)h(int)f(maxdims,)g(int)g(*dims,)g(int)h (*periods,)393 1091 y(int)g(*coords\))75 1198 y(MPI)p 150 1198 V 17 w(GET)p 239 1198 V 17 w(CART\(COMM,)e(MAXDIMS,)h(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 1255 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(PERIODS\(*\),)f(COORDS\(*\),)h(IERROR)166 1362 y FC(F)l(or)11 b(a)g(pro)q(cess)h(group)f(with)h(Cartesian)g (structure,)f(the)h(function)h Fv(MPI)p 1399 1362 14 2 v 15 w(CART)p 1533 1362 V 17 w(RANK)f FC(translates)75 1418 y(the)21 b(logical)h(pro)q(cess)f(co)q(ordinates)g(to)f(pro)q (cess)h(ranks)f(as)g(they)h(are)f(used)i(b)o(y)e(the)h(p)q(oin)o (t{to{p)q(oin)o(t)75 1475 y(routines:)75 1599 y Fv(MPI)p 160 1599 V 16 w(CART)p 295 1599 V 17 w(RANK\(comm,)13 b(co)q(o)o(rds,)i(rank\))117 1697 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j (\(han-)905 1753 y(dle\))117 1869 y(IN)171 b Fv(co)q(o)o(rds)449 b Fq(in)o(teger)15 b(arra)o(y)g(\(of)f(size)h Fk(ndims)p Fq(\))f(sp)q(ecifying)g(the)i(Cartesian)905 1926 y(co)q(ordinates)f(of) e(a)h(pro)q(cess)117 2042 y(OUT)124 b Fv(rank)488 b Fq(rank)14 b(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 2187 y Fo(int)23 b(MPI)p 245 2187 15 2 v 17 w(Cart)p 358 2187 V 17 w(rank\(MPI)p 567 2187 V 16 w(comm)g(comm,)g(int)h (*coords,)f(int)g(*rank\))75 2294 y(MPI)p 150 2294 V 17 w(CART)p 263 2294 V 16 w(RANK\(COMM,)g(COORDS,)g(RANK,)g(IERROR\)) 170 2350 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h(IERROR)166 2457 y FC(If)14 b(for)f(dimension)j Fo(i)e FC(with)g Fo(periods\(i\))22 b(=)i(true)13 b FC(the)h(co)q(ordinate)h Fo(coords\(i\))d FC(is)j(out)e(of)h(range,)75 2514 y(i.e.,)39 b Fo(coords\(i\))22 b Fw(<)j Fo(0)34 b FC(or)g Fo(coords\(i\))22 b FA(\025)i Fo(dims\(i\))p FC(,)38 b(it)d(is)g(shifted)g(bac)o(k)f(to)f (the)i(in)o(terv)m(al)75 2570 y Fo(0)24 b FA(\024)g Fo(coords\(i\))f FA(\024)h Fo(dims\(i\))c FC(automatically)l(.)40 b(Out{of{range)21 b(co)q(ordinates)h(are)f(erroneous)h(for)75 2627 y(ap)q(erio)q(dic)17 b(dimensions.)166 2704 y(The)e(in)o(v)o(erse)h(rank-to-co)q(ordinates)f (translation)g(is)h(pro)o(vided)g(b)o(y)f(the)g(function:)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 147 155 147 154 bop 75 -100 a Fs(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FC(147)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(CART)p 295 45 V 17 w(COORDS\(comm,)14 b(rank,)h(maxdims,)e(co)q(o)o(rds\))117 124 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j(\(han-)905 180 y(dle\))117 259 y(IN)171 b Fv(rank)488 b Fq(rank)14 b(of)f(a)h(pro)q(cess)i(within)d(group)g(of)h Fl(comm)e Fq(\(in)o(teger\))117 337 y(IN)171 b Fv(maxdims)403 b Fq(length)11 b(of)f(v)o(ector)i Fk(coord)d Fq(in)h(the)i(calling)d (program)g(\(in)o(teger\))117 416 y(OUT)124 b Fv(co)q(o)o(rds)449 b Fq(in)o(teger)14 b(arra)o(y)f(\(of)g(size)h Fk(ndims)p Fq(\))e(con)o(taining)h(the)h(Cartesian)905 472 y(co)q(ordinates)h(of)e (sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 599 y Fo(int)23 b(MPI)p 245 599 15 2 v 17 w(Cart)p 358 599 V 17 w(coords\(MPI)p 615 599 V 16 w(comm)g(comm,)g(int)h(rank,)f(int)g (maxdims,)g(int)g(*coords\))75 687 y(MPI)p 150 687 V 17 w(CART)p 263 687 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g(COORDS,)f (IERROR\))170 743 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f(COORDS\(*\),)h (IERROR)166 831 y FC(The)14 b(follo)o(wing)h(functions)h(pro)o(vide)e (\\adjacency")h(information)f(for)g(a)g(general)g(graph)g(top)q(ology)l (.)75 937 y Fv(MPI)p 160 937 14 2 v 16 w(GRAPH)p 326 937 V 17 w(NEIGHBORS)p 594 937 V 19 w(COUNT\(comm,)f(rank,)i(nneighb)q (o)o(rs\))117 1016 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)12 b(for)i(a)g(group)g(with)g(graph)g(top)q(ology)f(\(han-)905 1072 y(dle\))117 1151 y(IN)171 b Fv(rank)488 b Fq(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(\(in)o(teger\))117 1229 y(OUT)124 b Fv(nneighb)q(o)o(rs)369 b Fq(n)o(um)o(b)q(er)13 b(of)h(neigh)o(b)q(ors)g(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o (teger\))75 1355 y Fo(int)23 b(MPI)p 245 1355 15 2 v 17 w(Graph)p 382 1355 V 17 w(neighbors)p 615 1355 V 16 w(count\(MPI)p 847 1355 V 16 w(comm)g(comm,)g(int)h(rank,)f(int)g (*nneighbors\))75 1443 y(MPI)p 150 1443 V 17 w(GRAPH)p 287 1443 V 16 w(NEIGHBORS)p 519 1443 V 16 w(COUNT\(COMM,)g(RANK,)g (NNEIGHBORS,)f(IERROR\))170 1500 y(INTEGER)h(COMM,)h(RANK,)f (NNEIGHBORS,)f(IERROR)75 1635 y Fv(MPI)p 160 1635 14 2 v 16 w(GRAPH)p 326 1635 V 17 w(NEIGHBORS\(comm,)15 b(rank,)g(maxneighb)q(o)o(rs,)f(neighb)q(o)o(rs\))117 1714 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)12 b(for)i(a)g(group)g(with)g(graph)g(top)q(ology)f(\(han-)905 1770 y(dle\))117 1849 y(IN)171 b Fv(rank)488 b Fq(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(\(in)o(teger\))117 1927 y(OUT)124 b Fv(maxneighb)q(o)o(rs)312 b Fq(size)15 b(of)e(arra)o(y)h Fl(neighb)q(o)o(rs)h Fq(\(in)o(teger\))117 2006 y(OUT)124 b Fv(neighb)q(o)o(rs)392 b Fq(ranks)13 b(of)g(pro)q(cesses)i(that)e (are)h(neigh)o(b)q(ors)f(to)g(sp)q(eci\014ed)h(pro-)905 2062 y(cess)i(\(arra)o(y)d(of)h(in)o(teger\))75 2189 y Fo(int)23 b(MPI)p 245 2189 15 2 v 17 w(Graph)p 382 2189 V 17 w(neighbors\(MPI)p 711 2189 V 15 w(comm)g(comm,)h(int)f (rank,)g(int)h(*maxneighbors,)393 2245 y(int)g(*neighbors\))75 2333 y(MPI)p 150 2333 V 17 w(GRAPH)p 287 2333 V 16 w(NEIGHBORS\(COMM,)e (RANK,)h(MAXNEIGHBORS,)f(NEIGHBORS,)h(IERROR\))170 2390 y(INTEGER)g(COMM,)h(RANK,)f(MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR) 166 2478 y FC(Example:)k(Supp)q(ose)19 b(that)f Fo(comm)f FC(is)i(a)e(comm)o(unicator)h(for)g(a)f(group)h(with)h(a)e(sh)o (u\017e-exc)o(hange)75 2534 y(top)q(ology:)h(The)13 b(group)f(has)g(2) 596 2518 y Fh(n)631 2534 y FC(mem)o(b)q(ers.)19 b(Eac)o(h)12 b(pro)q(cess)h Fw(a)1130 2541 y Fi(1)1149 2534 y Fw(;)8 b(:)g(:)g(:)d(;)j(a)1275 2541 y Fh(n)1310 2534 y FC(\()p Fw(a)1352 2541 y Fh(i)1378 2534 y FA(2)13 b(f)p FC(0)p Fw(;)8 b FC(1)p FA(g)p FC(\))i(has)i(three)h(neigh-)75 2591 y(b)q(ors:)k(exc)o(hange\()p Fw(a)411 2598 y Fi(1)431 2591 y Fw(;)8 b(:)g(:)g(:)d(;)j(a)557 2598 y Fh(n)580 2591 y FC(\))k(=)h Fw(a)682 2598 y Fi(1)702 2591 y Fw(;)8 b(:)g(:)g(:)t(;)g(a)827 2598 y Fh(n)p Fj(\000)p Fi(1)895 2591 y Fw(;)g FC(\026)-23 b Fw(a)940 2598 y Fh(n)973 2591 y FC(\()q(\026)f Fw(a)12 b FC(=)h(1)p FA(\000)p Fw(a)p FC(\),)e(sh)o(u\017e\()p Fw(a)1366 2598 y Fi(1)1385 2591 y Fw(;)d(:)g(:)g(:)d(;)j(a)1511 2598 y Fh(n)1534 2591 y FC(\))k(=)h Fw(a)1636 2598 y Fi(2)1656 2591 y Fw(;)8 b(:)g(:)g(:)d(;)j(a)1782 2598 y Fh(n)1805 2591 y Fw(;)g(a)1850 2598 y Fi(1)1868 2591 y FC(,)75 2647 y(and)17 b(unsh)o(u\017e\()p Fw(a)382 2654 y Fi(1)402 2647 y Fw(;)8 b(:)g(:)g(:)d(;)j(a)528 2654 y Fh(n)551 2647 y FC(\))15 b(=)h Fw(a)659 2654 y Fh(n)683 2647 y Fw(;)8 b(a)728 2654 y Fi(1)747 2647 y Fw(;)g(:)g(:)g(:)d(;)j(a)873 2654 y Fh(n)p Fj(\000)p Fi(1)941 2647 y FC(.)25 b(The)17 b(graph)g(adjacency)g(list)h(is)g(illustrated)g(b)q(elo)o(w)75 2704 y(for)d Fw(n)d FC(=)h(3)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 148 156 148 155 bop 75 -100 a FC(148)882 b Fs(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)p 75 5 1119 2 v 74 62 2 57 v 133 45 a Fx(no)q(de)p 300 62 V 104 w(exc)o(hange)122 b(sh)o(u\017e)h(unsh)o(u\017e)p 1193 62 V 74 118 V 300 118 V 326 101 a FC(neigh)o(b)q(ors\(1\))49 b(neigh)o(b)q(ors\(2\))h (neigh)o(b)q(ors\(3\))p 1193 118 V 75 120 1119 2 v 74 176 2 57 v 100 159 a(0)f(\(000\))p 300 176 V 161 w(1)275 b(0)g(0)p 1193 176 V 74 233 V 100 216 a(1)49 b(\(001\))p 300 233 V 161 w(0)275 b(2)g(4)p 1193 233 V 74 289 V 100 272 a(2)49 b(\(010\))p 300 289 V 161 w(3)275 b(4)g(1)p 1193 289 V 74 346 V 100 329 a(3)49 b(\(011\))p 300 346 V 161 w(2)275 b(6)g(5)p 1193 346 V 74 402 V 100 385 a(4)49 b(\(100\))p 300 402 V 161 w(5)275 b(1)g(2)p 1193 402 V 74 459 V 100 442 a(5)49 b(\(101\))p 300 459 V 161 w(4)275 b(3)g(6)p 1193 459 V 74 515 V 100 498 a(6)49 b(\(110\))p 300 515 V 161 w(7)275 b(5)g(3)p 1193 515 V 74 571 V 100 555 a(7)49 b(\(111\))p 300 571 V 161 w(6)275 b(7)g(7)p 1193 571 V 75 573 1119 2 v 75 651 a Fo(C)48 b(assume:)22 b(each)i(process)f(has)g(stored)g(a)h(real)f(number)g(A.)75 707 y(C)48 b(extract)22 b(neighborhood)h(information)218 764 y(CALL)g(MPI_COMM_RANK\(comm,)f(myrank,)h(ierr\))218 820 y(CALL)g(MPI_GRAPH_NEIGHBORS\(comm,)e(myrank,)i(3,)g(neighbors,)g (ierr\))75 877 y(C)48 b(perform)22 b(exchange)h(permutation)218 933 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e (neighbors\(1\),)h(0,)194 990 y(+)119 b(neighbors\(1\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 1046 y(C)48 b(perform)22 b(shuffle)h(permutation)218 1103 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e (1,)j(MPI_REAL,)e(neighbors\(2\),)h(0,)194 1159 y(+)119 b(neighbors\(3\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 1215 y(C)48 b(perform)22 b(unshuffle)h(permutation)218 1272 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e (neighbors\(3\),)h(0,)194 1328 y(+)119 b(neighbors\(2\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 1450 y Fp(6.5.2)49 b(Ca)o(rtesian)16 b(shift)g(co)q(o)o(rdinates)75 1535 y FC(If)i(the)f(pro)q(cess)h(top)q(ology)f(is)h(a)f(Cartesian)g (structure,)h(a)f(collectiv)o(e)i(shift)f(op)q(eration)g(is)g(most)e (lik)o(ely)75 1592 y(to)g(b)q(e)g(used)h(along)f(a)g(co)q(ordinate)h (direction.)24 b(In)17 b(MPI)f(shift)g(op)q(erations)h(are)e(handled)j (b)o(y)e(the)g(more)75 1648 y(general)k(send{receiv)o(e)i(functions.)35 b(As)20 b(input)h(they)f(tak)o(e)f(the)h(rank)g(of)f(a)h(source)g(pro)q (cess)g(for)g(the)75 1705 y(receiv)o(e,)c(and)f(the)g(rank)g(of)g(a)g (destination)h(pro)q(cess)g(for)e(the)h(send.)166 1761 y(If)j(the)g(function)h Fv(MPI)p 561 1761 14 2 v 16 w(CART)p 696 1761 V 17 w(SHIFT)f FC(is)g(called)i(for)d(a)h(Cartesian)g(pro)q (cess)g(group,)g(it)g(pro)o(vides)75 1818 y(the)c(calling)i(pro)q(cess) e(with)h(the)f(ab)q(o)o(v)o(e)g(iden)o(ti\014ers,)h(whic)o(h)g(then)g (can)f(b)q(e)h(passed)f(to)f(the)i(send{receiv)o(e)75 1874 y(function.)21 b(The)15 b(user)g(sp)q(eci\014es)i(the)e(co)q (ordinate)h(direction)g(for)f(the)g(shift)h(and)f(the)g(step)g(length.) 21 b(The)75 1931 y(function)16 b(is)g(lo)q(cal.)75 2034 y Fv(MPI)p 160 2034 V 16 w(CART)p 295 2034 V 17 w(SHIFT\(comm,)c (direction,)k(disp,)g(rank)p 977 2034 V 16 w(source,)g(rank)p 1224 2034 V 16 w(dest\))117 2111 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j (\(han-)905 2168 y(dle\))117 2243 y(IN)171 b Fv(direction)408 b Fq(co)q(ordinate)14 b(dimension)f(of)g(shift)h(\(in)o(teger\))117 2317 y(IN)171 b Fv(disp)497 b Fq(displacemen)o(t)18 b(\()p Fa(>)h Fq(0:)27 b(up)o(w)o(ards)18 b(shift,)h Fa(<)g Fq(0:)27 b(do)o(wn)o(w)o(ards)905 2374 y(shift\))14 b(\(in)o(teger\)) 117 2448 y(OUT)124 b Fv(rank)p 420 2448 V 16 w(source)353 b Fq(rank)14 b(of)f(source)j(pro)q(cess)f(\(in)o(teger\))117 2523 y(OUT)124 b Fv(rank)p 420 2523 V 16 w(dest)396 b Fq(rank)14 b(of)f(destination)h(pro)q(cess)i(\(in)o(teger\))75 2647 y Fo(int)23 b(MPI)p 245 2647 15 2 v 17 w(Cart)p 358 2647 V 17 w(shift\(MPI)p 591 2647 V 16 w(comm)g(comm,)g(int)h (direction,)e(int)i(disp,)f(int)g(*rank)p 1681 2647 V 17 w(source,)393 2704 y(int)h(*rank)p 612 2704 V 16 w(dest\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 149 157 149 156 bop 75 -100 a Fs(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b FC(149)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(CART)p 263 45 V 16 w(SHIFT\(COMM,)23 b(DIRECTION,)f(DISP,)h(RANK)p 1067 45 V 17 w(SOURCE,)g(RANK)p 1371 45 V 17 w(DEST,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(DIRECTION,)e(DISP,)h(RANK)p 1009 102 V 17 w(SOURCE,)g(RANK)p 1313 102 V 16 w(DEST,)h(IERROR)166 188 y FC(Dep)q(ending)17 b(on)e(the)g(p)q(erio)q(dicit)o(y)i(of)e(the)g (Cartesian)g(group)g(in)h(the)f(sp)q(eci\014ed)i(co)q(ordinate)f (direc-)75 245 y(tion,)d Fv(MPI)p 264 245 14 2 v 16 w(CART)p 399 245 V 17 w(SHIFT)f FC(pro)o(vides)h(the)g(iden)o(ti\014ers)h(for)e (a)h(circular)h(or)e(an)g(end{o\013)h(shift.)19 b(In)13 b(the)g(case)75 301 y(of)j(an)h(end{o\013)g(shift,)g(the)g(v)m(alue)h Fl(MPI)p 746 301 13 2 v 14 w(PROCNULL)e FC(ma)o(y)g(b)q(e)i(returned)f (in)g Fv(rank)p 1478 301 14 2 v 17 w(source)g FC(or)f Fv(rank)p 1771 301 V 17 w(dest)p FC(,)75 358 y(indicating)h(that)e(the) g(source)g(or)g(the)g(destination)h(for)f(the)g(shift)h(is)f(out)g(of)g (range.)166 414 y(Example:)37 b(The)24 b(comm)o(unicator)f Fv(comm)f FC(is)i(asso)q(ciated)f(with)h(a)f(t)o(w)o(o-dimensional)i(p) q(erio)q(dic)75 471 y(Cartesian)14 b(top)q(ology)l(.)19 b(A)14 b(t)o(w)o(o-dimensional)i(arra)o(y)c(of)i Fo(REAL)p FC(s)g(is)g(stored)g(one)g(elemen)o(t)h(p)q(er)g(pro)q(cess,)f(in)75 527 y(v)m(ariable)j Fo(A)p FC(.)d(One)i(wishes)g(to)f(sk)o(ew)f(this)i (arra)o(y)l(,)e(b)o(y)h(shifting)h(column)g Fw(i)f FC(b)o(y)g Fw(i)p FC(.)75 622 y Fo(....)75 678 y(C)24 b(find)f(process)g(rank)218 735 y(CALL)g(MPI_COMM_RANK\(comm,)f(rank,)h(ierr,)g(ierr\)\))75 791 y(C)h(find)f(Cartesian)g(coordinates)218 848 y(CALL)g (MPI_CART_COORD\(comm,)f(rank,)h(coords,)g(ierr\))75 904 y(C)h(compute)f(shift)g(source)g(and)g(destination)218 961 y(CALL)g(MPI_CART_SHIFT\(comm,)f(2,)h(coords\(2\),)g(source,)g (dest,)g(ierr\))75 1017 y(C)h(skew)f(array)218 1074 y(CALL)g (MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(dest,)i(0,)f(source,)g(0,) h(comm,)194 1130 y(+)621 b(status,)23 b(ierr\))75 1253 y Fp(6.5.3)49 b(P)o(a)o(rtitioning)17 b(of)g(Ca)o(rtesian)e(structures) 75 1339 y FC(If)40 b(a)f(Cartesian)g(top)q(ology)g(has)g(b)q(een)i (created)e(with)h Fv(MPI)p 1280 1339 V 16 w(MAKE)p 1425 1339 V 16 w(CART)p FC(,)g(the)f(function)75 1396 y Fv(MPI)p 160 1396 V 16 w(CART)p 295 1396 V 17 w(SUB)20 b FC(can)f(b)q(e)h(used)g (to)e(partition)i(the)f(comm)o(unicator)g(group)g(in)o(to)g(subgroups)g (whic)o(h)75 1452 y(form)d(lo)o(w)o(er{dimensional)h(Cartesian)f (subgrids,)h(and)g(to)f(build)i(for)d(eac)o(h)i(subgroup)f(a)g(comm)o (unica-)75 1509 y(tor)e(with)i(the)f(asso)q(ciated)h(subgrid)g (Cartesian)f(top)q(ology)l(.)75 1613 y Fv(MPI)p 160 1613 V 16 w(CART)p 295 1613 V 17 w(SUB\(comm,)e(remain)p 688 1613 V 16 w(dims,)h(comm)p 936 1613 V 15 w(new\))117 1690 y Fq(IN)171 b Fv(comm)454 b Fq(comm)o(unicator)7 b(for)i(group)g(with)g(Cartesian)g(structure)j(\(han-)905 1746 y(dle\))117 1822 y(IN)171 b Fv(remain)p 466 1822 V 15 w(dims)339 b Fq(the)22 b Fk(i)p Fq(-th)f(en)o(try)g(of)g Fl(remain)p 1369 1822 13 2 v 15 w(dims)f Fq(sp)q(eci\014es)j(whether)f (the)905 1878 y Fk(i)p Fq(-th)g(dimension)e(is)i(k)o(ept)g(in)f(the)h (subgrid)g(\()p Fk(true)p Fq(\))f(or)h(is)905 1935 y(dropp)q(ed)15 b(\()p Fk(false)p Fq(\))e(\(logical)f(v)o(ector\))117 2010 y(OUT)124 b Fv(comm)p 454 2010 14 2 v 14 w(new)366 b Fq(comm)o(unicator)14 b(con)o(taining)h(the)i(subgrid)g(whic)o(h)f (includes)905 2067 y(the)f(calling)d(pro)q(cess)k(\(handle\))75 2192 y Fo(int)23 b(MPI)p 245 2192 15 2 v 17 w(Cart)p 358 2192 V 17 w(sub\(MPI)p 543 2192 V 16 w(comm)g(comm,)h(int)f (*remain)p 1085 2192 V 16 w(dims,)g(MPI)p 1316 2192 V 17 w(comm)h(*comm)p 1573 2192 V 16 w(new\))75 2278 y(MPI)p 150 2278 V 17 w(CART)p 263 2278 V 16 w(SUB\(COMM,)f(REMAIN)p 662 2278 V 16 w(DIMS,)h(COMM)p 918 2278 V 16 w(NEW,)g(IERROR\))170 2335 y(INTEGER)f(COMM,)h(REMAIN)p 652 2335 V 16 w(DIMS\(*\),)f(COMM)p 979 2335 V 16 w(NEW,)h(IERROR)166 2421 y FC(Example:)19 b(Assume)12 b(that)f Fv(MPI)p 716 2421 14 2 v 16 w(MAKE)p 861 2421 V 28 w(CART\(...,)h(comm\))d FC(has)j(de\014ned)i(a)d(\(2)t FA(\002)t FC(3)t FA(\002)t FC(4\))g(grid.)19 b(Let)75 2478 y Fo(remain)p 222 2478 15 2 v 16 w(dims)24 b(=)f(\(true,)g(false,) g(true\))p FC(.)18 b(Then)11 b(a)e(call)j(to)d Fv(MPI)p 1247 2478 14 2 v 16 w(CART)p 1382 2478 V 17 w(SUB\(comm,)g(remain)p 1771 2478 V 16 w(dims,)75 2534 y(comm)p 195 2534 V 15 w(new\))14 b FC(will)h(create)f(comm)o(unicators)f(for)h(three)g(new)g (groups)f(with)h(eigh)o(t)h(pro)q(cesses,)f(eac)o(h)g(with)75 2591 y(a)21 b(2)13 b FA(\002)h FC(4)21 b(Cartesian)f(top)q(ology)l(.)37 b(If)21 b Fo(remain)p 874 2591 15 2 v 16 w(dims)j(=)g(\(false,)e (false,)h(true\))e FC(then)g(the)g(call)h(to)75 2647 y Fv(MPI)p 160 2647 14 2 v 16 w(CART)p 295 2647 V 17 w(SUB\(comm,)h(remain)p 698 2647 V 16 w(dims,)h(comm)p 956 2647 V 15 w(new\))f FC(will)i(create)e(comm)o(unicators)g(for)f (six)i(new)75 2704 y(groups)15 b(with)g(four)g(pro)q(cesses,)g(eac)o(h) h(with)f(a)g(one-dimensional)j(Cartesian)d(top)q(ology)l(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 150 158 150 157 bop 75 -100 a FC(150)882 b Fs(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fp(6.5.4)49 b(Lo)o(w{level)18 b(top)q(ology)g(functions)75 199 y FC(The)11 b(t)o(w)o(o)e(new)h (functions)i(in)o(tro)q(duced)f(in)h(this)f(section)g(can)f(b)q(e)h (used)g(to)f(implemen)o(t)i(all)f(other)f(top)q(ology)75 256 y(functions.)21 b(In)15 b(general)h(they)f(will)i(not)e(b)q(e)h (called)h(b)o(y)e(the)g(user)g(directly)l(.)75 394 y Fv(MPI)p 160 394 14 2 v 16 w(MAP)p 275 394 V 16 w(CART\(comm)p 545 394 V 15 w(old,)g(ndims,)g(dims,)g(p)q(erio)q(ds,)h(newrank\))117 507 y Fq(IN)171 b Fv(comm)p 454 507 V 14 w(old)383 b Fq(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g(\(handle\))117 652 y(IN)171 b Fv(ndims)460 b Fq(n)o(um)o(b)q(er)12 b(of)f(dimensions)h (of)f(Cartesian)i(structure)h(\(in)o(teger\))117 797 y(IN)171 b Fv(dims)483 b Fq(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fk(ndims)e Fq(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 854 y(pro)q(cesses)g(in)c(eac)o(h)i(co)q(ordinate)f(direction)117 999 y(IN)171 b Fv(p)q(erio)q(ds)436 b Fq(logical)16 b(arra)o(y)h(of)g (size)h Fk(ndims)f Fq(sp)q(ecifying)g(the)h(p)q(erio)q(dicit)o(y)905 1056 y(sp)q(eci\014cation)d(in)e(eac)o(h)i(co)q(ordinate)f(direction) 117 1201 y(OUT)124 b Fv(newrank)414 b Fq(reordered)11 b(rank)f(of)e(the)i(calling)e(pro)q(cess;)k Fl(MPI)p 1643 1201 13 2 v 15 w(UNDEFINED)905 1257 y Fq(if)h(calling)g(pro)q (cess)i(do)q(es)g(not)f(b)q(elong)f(to)h(grid)g(\(in)o(teger\))75 1417 y Fo(int)23 b(MPI)p 245 1417 15 2 v 17 w(Map)p 334 1417 V 17 w(cart\(MPI)p 543 1417 V 16 w(comm)g(comm)p 774 1417 V 17 w(old,)g(int)h(ndims,)f(int)g(*dims,)g(int)h(*period,)393 1473 y(int)g(*newrank\))75 1595 y(MPI)p 150 1595 V 17 w(MAP)p 239 1595 V 17 w(CART\(COMM)p 472 1595 V 15 w(OLD,)g(NDIMS,)f (DIMS,)g(PERIOD,)g(NEWRANK,)g(IERROR\))170 1651 y(INTEGER)g(COMM)p 460 1651 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(PERIOD\(*\),)g(NEWRANK,)f (IERROR)166 1773 y FC(This)c(function)g(computes)g(an)g(optimal)g (placemen)o(t)g(for)f(the)g(calling)j(pro)q(cess)e(on)f(the)h(ph)o (ysical)75 1829 y(mac)o(hine.)31 b(A)19 b(p)q(ossible)i(implemen)o (tation)f(of)e(this)h(function)h(is)f(to)f(alw)o(a)o(ys)g(return)h(the) g(rank)g(of)f(the)75 1886 y(calling)f(pro)q(cess,)e(i.e.,)g(not)g(to)f (p)q(erform)h(an)o(y)g(reordering.)166 1977 y(The)k(function)g Fv(MPI)p 529 1977 14 2 v 16 w(MAKE)p 674 1977 V 16 w(CART\(comm)p 944 1977 V 15 w(old,)g(ndims,)g(dims,)g(p)q(erio)q(ds,)i(reo)o(rder,)c (comm)p 1760 1977 V 15 w(ca)o(rt\))p FC(,)75 2034 y(with)j Fo(reorder)j(=)g(true)c FC(can)h(b)q(e)g(implemen)o(ted)h(b)o(y)f (calling)h Fv(MPI)p 1264 2034 V 16 w(MAP)p 1379 2034 V 16 w(CART\(comm)p 1649 2034 V 15 w(old,)f(ndims,)75 2090 y(dims,)13 b(p)q(erio)q(ds,)i(newrank\))p FC(,)f(then)g(calling)h Fv(MPI)p 883 2090 V 16 w(COMM)p 1041 2090 V 16 w(SPLIT\(comm,)c(colo)o (r,)h(k)o(ey)l(,)i(comm)p 1658 2090 V 15 w(ca)o(rt\))p FC(,)f(with)75 2147 y Fo(color)23 b(=)h(0)17 b FC(if)h Fo(newrank)23 b FA(6)p FC(=)h Fo(MPI)p 676 2147 15 2 v 17 w(UNDEFINED)p FC(,)16 b Fo(color)23 b(=)h(MPI)p 1201 2147 V 17 w(UNDEFINED)16 b FC(otherwise,)i(and)g Fo(key)23 b(=)75 2203 y(newrank)p FC(.)166 2295 y(The)15 b(function)g Fv(MPI)p 521 2295 14 2 v 16 w(CART)p 656 2295 V 17 w(SUB\(comm,)e(remain)p 1049 2295 V 15 w(dims,)h(comm)p 1296 2295 V 15 w(new\))h FC(can)f(b)q(e)i(implemen)o(ted)g(b)o(y)75 2351 y(a)f(call)i(to)e Fv(MPI)p 338 2351 V 15 w(COMM)p 495 2351 V 17 w(SPLIT\(comm,)d(colo)o(r,)i(k)o(ey)l(,)i(comm)p 1118 2351 V 14 w(new\))p FC(,)g(using)g(a)f(single)i(n)o(um)o(b)q(er)e (enco)q(ding)75 2408 y(of)g(the)g(lost)g(dimensions)i(as)e Fo(color)g FC(and)g(a)g(single)i(n)o(um)o(b)q(er)f(enco)q(ding)g(of)f (the)h(preserv)o(ed)f(dimensions)75 2464 y(as)g Fo(key)p FC(.)166 2556 y(All)h(other)e(Cartesian)g(top)q(ology)g(functions)h (can)g(b)q(e)g(implemen)o(ted)h(lo)q(cally)l(,)h(using)e(the)f(top)q (ology)75 2612 y(information)h(that)g(is)h(cac)o(hed)f(with)h(the)f (comm)o(unicator.)166 2704 y(The)g(corresp)q(onding)i(new)e(function)h (for)f(general)g(graph)g(structures)g(is)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 151 159 151 158 bop 75 -100 a Fs(6.6.)34 b(A)15 b(SIMPLE)h(EXAMPLE)1157 b FC(151)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(MAP)p 275 45 V 16 w(GRAPH\(comm)p 576 45 V 16 w(old,)15 b(nno)q(des,)h(index,)g (edges,)g(newrank\))117 122 y Fq(IN)171 b Fv(comm)p 454 122 V 14 w(old)383 b Fq(comm)o(unicator)11 b(for)j(paren)o(t)g(group)g (\(handle\))117 197 y(IN)171 b Fv(nno)q(des)441 b Fq(n)o(um)o(b)q(er)13 b(of)h(graph)f(no)q(des)i(\(in)o(teger\))117 273 y(IN)171 b Fv(index)473 b Fq(in)o(teger)30 b(arra)o(y)g(sp)q(ecifying)f(the)i (graph)e(structure,)36 b(see)905 329 y Fl(MPI)p 982 329 13 2 v 15 w(MAKE)p 1115 329 V 14 w(GRAPH)117 404 y Fq(IN)171 b Fv(edges)468 b Fq(in)o(teger)15 b(arra)o(y)e(sp)q(ecifying)h(the)g (graph)g(structure)117 479 y(OUT)124 b Fv(newrank)414 b Fq(reordered)11 b(rank)f(of)e(the)i(calling)e(pro)q(cess;)k Fl(MPI)p 1643 479 V 15 w(UNDEFINED)905 536 y Fq(if)j(the)i(calling)d (pro)q(cess)k(do)q(es)e(not)g(b)q(elong)g(to)f(graph)h(\(in)o(te-)905 592 y(ger\))75 717 y Fo(int)23 b(MPI)p 245 717 15 2 v 17 w(Map)p 334 717 V 17 w(graph\(MPI)p 567 717 V 16 w(comm)g(comm)p 798 717 V 17 w(old,)g(int)h(nnodes,)f(int)g(*index,)g(int)g(*edges,)393 773 y(int)h(*newrank\))75 860 y(MPI)p 150 860 V 17 w(MAP)p 239 860 V 17 w(GRAPH\(COMM)p 496 860 V 15 w(OLD,)g(NNODES,)f(INDEX,)g (EDGES,)g(NEWRANK,)f(IERROR\))170 916 y(INTEGER)h(COMM)p 460 916 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g(EDGES\(*\),)f(NEWRANK,)h (IERROR)166 1002 y FC(The)10 b(function)h Fv(MPI)p 512 1002 14 2 v 16 w(MAKE)p 657 1002 V 16 w(GRAPH\(comm)p 958 1002 V 16 w(old,)f(nno)q(des,)j(index,)f(edges,)g(reo)o(rder,)d (comm)p 1733 1002 V 15 w(graph\))p FC(,)75 1059 y(with)k Fo(reorder)23 b(=)h(true)12 b FC(can)h(b)q(e)h(implemen)o(ted)h(b)o(y)e (calling)i Fv(MPI)p 1219 1059 V 15 w(MAP)p 1333 1059 V 17 w(GRAPH\(comm)p 1635 1059 V 15 w(old,)e(nno)q(des,)75 1115 y(index,)g(edges,)g(newrank\))p FC(,)g(then)f(calling)h Fv(MPI)p 854 1115 V 16 w(COMM)p 1012 1115 V 17 w(SPLIT\(comm,)c(colo)o (r,)i(k)o(ey)l(,)h(comm)p 1625 1115 V 15 w(graph\))p FC(,)g(with)75 1172 y Fo(color)23 b(=)h(0)17 b FC(if)h Fo(newrank)23 b FA(6)p FC(=)h Fo(MPI)p 676 1172 15 2 v 17 w(UNDEFINED)p FC(,)16 b Fo(color)23 b(=)h(MPI)p 1201 1172 V 17 w(UNDEFINED)16 b FC(otherwise,)i(and)g Fo(key)23 b(=)75 1228 y(newrank)p FC(.)166 1285 y(All)f(other)f(graph)g (top)q(ology)g(functions)h(can)g(b)q(e)g(implemen)o(ted)h(lo)q(cally)l (,)h(using)e(the)f(top)q(ology)75 1341 y(information)15 b(that)g(is)h(cac)o(hed)f(with)h(the)f(comm)o(unicator.)75 1484 y Ft(6.6)59 b(A)20 b(simple)f(example)75 1586 y FC(The)12 b(example)h(in)g(\014gure)f(6.1)f(sho)o(ws)g(ho)o(w)h(the)g (grid)g(de\014nition)i(and)e(inquiry)i(functions)f(can)f(b)q(e)h(used)f (in)75 1642 y(an)k(application)h(program.)k(A)16 b(partial)g (di\013eren)o(tial)h(equation,)f(for)f(example)h(the)g(P)o(oisson)g (equation,)75 1699 y(is)g(to)f(b)q(e)h(solv)o(ed)f(on)h(a)f (rectangular)g(domain.)21 b(First)15 b(the)g(pro)q(cesses)h(organize)f (themselv)o(es)h(in)h(a)e(t)o(w)o(o{)75 1755 y(dimensional)h (structure.)k(Eac)o(h)14 b(pro)q(cess)h(then)f(inquires)j(for)c(the)i (ranks)f(of)g(its)g(neigh)o(b)q(ors)i(in)f(the)f(four)75 1812 y(directions.)29 b(The)18 b(n)o(umerical)h(problem)g(is)f(solv)o (ed)h(b)o(y)f(an)f(iterativ)o(e)h(metho)q(d,)h(the)f(details)h(of)e (whic)o(h)75 1868 y(are)e(hidden)i(in)f(the)f(subroutine)h Fo(relax)p FC(.)166 1925 y(In)e(eac)o(h)f(relaxation)g(step)g(eac)o(h)g (pro)q(cess)h(computes)f(new)g(v)m(alues)h(for)f(the)g(solution)h(grid) f(function)75 1981 y(at)i(all)i(p)q(oin)o(ts)f(o)o(wned)g(b)o(y)f(the)h (pro)q(cess.)22 b(Then)16 b(the)g(v)m(alues)h(at)e(in)o(terpro)q(cess)i (b)q(oundaries)g(ha)o(v)o(e)e(to)g(b)q(e)75 2038 y(exc)o(hanged)d(with) h(neigh)o(b)q(oring)g(pro)q(cesses.)19 b(F)l(or)11 b(example,)i(the)f (exc)o(hange)g(subroutine)h(migh)o(t)f(con)o(tain)75 2094 y(a)i(call)h(lik)o(e)g Fv(MPI)p 361 2094 14 2 v 15 w(SEND\(...,neigh)p 663 2094 V 16 w(rank\(1\),...\))i FC(to)c(send)i(up)q(dated)g(v)m(alues)g(to)e(the)h(left{hand)g(neigh)o (b)q(or)75 2150 y(\(i-1,j\).)19 b(Note)c(that)g(there)g(is)h(only)f(lo) q(cal)i(comm)o(unication)f(in)g(this)f(exc)o(hange)h(op)q(eration.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 152 160 152 159 bop 75 -100 a FC(152)882 b Fs(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)194 163 y Fo(integer)23 b(ndims,)g(num)p 627 163 15 2 v 17 w(neigh)194 219 y(logical)g(reorder) 194 276 y(parameter)g(\(ndims=2,)g(num)p 747 276 V 16 w(neigh=4,)g(reorder=.true.\))194 332 y(integer)g(comm,)g(comm)p 627 332 V 17 w(cart,)g(dims\(ndims\),)f(neigh)p 1217 332 V 17 w(def\(ndims\))194 388 y(integer)h(neigh)p 508 388 V 17 w(rank\(num)p 717 388 V 16 w(neigh\),)g(own)p 996 388 V 16 w(position\(ndims\),)f(i,)i(j)194 445 y(logical)f (periods\(ndims\))194 501 y(real)p FA(\003)p Fo(8)g(u\(0:101,0:101\),)f (f\(0:101,0:101\))194 558 y(data)i(dims)f(/)h(ndims)f FA(\003)g Fo(0)h(/)194 614 y(comm)g(=)f(MPI)p 436 614 V 17 w(COMM)p 549 614 V 17 w(WORLD)75 671 y(C)107 b(Set)23 b(process)g(grid)g(size)h(and)f(periodicity)194 727 y(call)h(MPI)p 389 727 V 16 w(MAKDIM\(comm,)f(ndims,)g(dims\))194 784 y(periods\(1\))g(=)h(.TRUE.)194 840 y(periods\(2\))f(=)h(.TRUE.)75 897 y(C)107 b(Create)23 b(a)g(grid)h(structure)e(in)i(WORLD)f(group)g (and)h(inquire)f(about)g(own)g(position)194 953 y(call)h(MPI)p 389 953 V 16 w(MAKE)p 501 953 V 17 w(CART)f(\(comm,)g(ndims,)g(dims,)h (periods,)e(reorder,)h(comm)p 1640 953 V 17 w(cart\))194 1009 y(call)h(MPI)p 389 1009 V 16 w(GET)p 477 1009 V 17 w(CART)f(\(comm)p 733 1009 V 17 w(cart,)g(ndims,)g(dims,)g(periods,) g(own)p 1490 1009 V 17 w(position\))75 1066 y(C)107 b(Look)23 b(up)h(the)f(ranks)g(for)h(the)f(neighbors.)46 b(Own)24 b(process)f(coordinates)f(are)i(\(i,j\).)75 1122 y(C)107 b(Neighbors)22 b(are)i(\(i-1,j\),)f(\(i+1,j\),)f(\(i,j-1\),)h (\(i,j+1\))194 1179 y(i)h(=)g(own)p 365 1179 V 17 w(position\(1\))194 1235 y(j)g(=)g(own)p 365 1235 V 17 w(position\(2\))194 1292 y(neigh)p 317 1292 V 17 w(def\(1\))f(=)h(i-1)194 1348 y(neigh)p 317 1348 V 17 w(def\(2\))f(=)h(j)194 1405 y(call)g(MPI)p 389 1405 V 16 w(CART)p 501 1405 V 17 w(RANK)f(\(comm)p 757 1405 V 17 w(cart,)g(neigh)p 1037 1405 V 17 w(def,)g(neigh)p 1293 1405 V 16 w(rank\(1\)\))194 1461 y(neigh)p 317 1461 V 17 w(def\(1\))g(=)h(i+1)194 1518 y(neigh)p 317 1518 V 17 w(def\(2\))f(=)h(j)194 1574 y(call)g(MPI)p 389 1574 V 16 w(CART)p 501 1574 V 17 w(RANK)f(\(comm)p 757 1574 V 17 w(cart,)g(neigh)p 1037 1574 V 17 w(def,)g(neigh)p 1293 1574 V 16 w(rank\(2\)\))194 1630 y(neigh)p 317 1630 V 17 w(def\(1\))g(=)h(i)194 1687 y(neigh)p 317 1687 V 17 w(def\(2\))f(=)h(j-1)194 1743 y(call)g(MPI)p 389 1743 V 16 w(CART)p 501 1743 V 17 w(RANK)f(\(comm)p 757 1743 V 17 w(cart,)g(neigh)p 1037 1743 V 17 w(def,)g(neigh)p 1293 1743 V 16 w(rank\(3\)\))194 1800 y(neigh)p 317 1800 V 17 w(def\(1\))g(=)h(i)194 1856 y(neigh)p 317 1856 V 17 w(def\(2\))f(=)h(j+1)194 1913 y(call)g(MPI)p 389 1913 V 16 w(CART)p 501 1913 V 17 w(RANK)f(\(comm)p 757 1913 V 17 w(cart,)g(neigh)p 1037 1913 V 17 w(def,)g(neigh)p 1293 1913 V 16 w(rank\(4\)\))75 1969 y(C)107 b(Initialize)22 b(the)i(grid)f(functions)g(and)g(start)g(the)h(iteration)194 2026 y(call)g(init)f(\(u,)g(f\))194 2082 y(do)h(10)f(it=1,100)242 2139 y(call)g(relax)h(\(u,)f(f\))75 2195 y(C)107 b(Exchange)22 b(data)i(with)f(neighbor)g(processes)242 2251 y(call)g(exchange)g(\(u,) h(comm)p 771 2251 V 16 w(cart,)f(neigh)p 1050 2251 V 17 w(rank,)g(num)p 1282 2251 V 17 w(neigh\))75 2308 y(10)71 b(continue)194 2364 y(call)24 b(output)f(\(u\))194 2421 y(end)147 2625 y FC(Figure)15 b(6.1:)k(Set{up)d(of)e(pro)q(cess)i (structure)f(for)f(t)o(w)o(o{dimensional)i(parallel)h(P)o(oisson)e (solv)o(er.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 153 161 153 160 bop 75 361 a Fu(Chapter)34 b(7)75 573 y Fy(MPI)40 b(En)m(vironmen)m(tal)h(Managemen)m(t)75 800 y FC(This)15 b(c)o(hapter)e(discusses)j(routines)e(for)g(getting)f(and,)i(where)f (appropriate,)g(setting)g(v)m(arious)g(parame-)75 856 y(ters)h(that)g(relate)g(to)g(the)g(MPI)h(implemen)o(tation)g(and)g (the)f(execution)i(en)o(vironmen)o(t.)j(\(suc)o(h)c(as)f(error)75 913 y(handling)i(and)f(comm)o(unication)g(bu\013ering\).)21 b(The)16 b(pro)q(cedures)g(for)f(en)o(tering)h(and)g(lea)o(ving)g(the)g (MPI)75 969 y(execution)g(en)o(vironmen)o(t)g(are)f(also)g(dexcrib)q (ed)i(here.)75 1138 y Ft(7.1)59 b(Implementation)17 b(info)n(rmation)75 1249 y Fp(7.1.1)49 b(Environmental)17 b(Inquiries)75 1344 y FC(The)e(follo)o(wing)h(routines)g(allo)o(w)f(to)g(inquire)i(on) e(the)g(execution)i(en)o(vironmen)o(t)e(of)g(an)g(MPI)g(program.)75 1452 y Fv(MPI)p 160 1452 14 2 v 16 w(GET)p 264 1452 V 17 w(V)l(ALID)p 408 1452 V 16 w(T)l(A)o(G)p 510 1452 V 17 w(RANGE)h(\()f(lo)o(w,)f(high)i(\))117 1533 y Fq(OUT)108 b Fv(lo)o(w)523 b Fq(Smallest)12 b(v)n(alid)h(tag)g(v)n(alue)h(\(in)o (teger\))117 1617 y(OUT)108 b Fv(high)507 b Fq(Largest)15 b(v)n(alid)d(tag)i(v)n(alue)f(\(in)o(teger\))75 1746 y Fo(int)23 b(MPI)p 245 1746 15 2 v 17 w(Get)p 334 1746 V 17 w(valid)p 471 1746 V 16 w(tag)p 559 1746 V 17 w(range\(int)g (*low,)g(int)h(*high\))75 1837 y(MPI)p 150 1837 V 17 w(GET)p 239 1837 V 17 w(VALID)p 376 1837 V 16 w(TAG)p 464 1837 V 17 w(RANGE\(LOW,)e(HIGH,)h(IERROR\))170 1893 y(INTEGER)g(LOW,)h(HIGH,)f(IERROR)75 1984 y FC(This)16 b(routine)g(gets)f(the)g(range)g(of)g(v)m(alid)i(tags;)e(that)f(is,)i (the)f(user)h(ma)o(y)f(use)h(an)o(y)f(tag)f(v)m(alue)j(from)d Fo(low)75 2041 y FC(to)h Fo(high)f FC(inclusiv)o(e.)75 2149 y Fv(MPI)p 160 2149 14 2 v 16 w(GET)p 264 2149 V 17 w(PROCESSOR)p 540 2149 V 18 w(NAME\()i(len,)f(name)g(\))117 2231 y Fq(IN)155 b Fv(len)533 b Fq(Length)14 b(\(in)g(c)o(haracters\))i (of)d(the)i(space)g(in)e Fk(name)117 2314 y Fq(OUT)108 b Fv(name)485 b Fq(A)17 b(unique)g(sp)q(eci\014er)i(for)e(the)g(actual) g(\(as)g(opp)q(osed)h(to)e(vir-)905 2371 y(tual\))e(no)q(de.)75 2500 y Fo(int)23 b(MPI)p 245 2500 15 2 v 17 w(Get)p 334 2500 V 17 w(processor)p 567 2500 V 16 w(name\(int)g(len,)g(char)g (*name\))75 2591 y(MPI)p 150 2591 V 17 w(GET)p 239 2591 V 17 w(PROCESSOR)p 472 2591 V 15 w(NAME\(NAME,)g(IERROR\))170 2647 y(CHARACTER*\(*\))g(NAME)170 2704 y(INTEGER)g(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 154 162 154 161 bop 75 -100 a FC(154)532 b Fs(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)166 45 y FC(The)j(name)g(is)g(a)f(c)o(haracter)g(string)h(for)f(maxim)o(um)h (\015exibilit)o(y)l(.)33 b(F)l(rom)18 b(this)h(v)m(alue)h(it)f(m)o(ust) f(b)q(e)75 102 y(p)q(ossible)23 b(to)d(iden)o(tify)i(a)e(sp)q(eci\014c) j(piece)f(of)e(hardw)o(are;)j(p)q(ossible)f(v)m(alues)g(include)h (\\pro)q(cessor)e(9)f(in)75 158 y(rac)o(k)f(4)h(of)f(mpp.cs.org")g(and) h(\\231")e(\(where)i(231)f(is)h(the)g(actual)g(pro)q(cessor)g(n)o(um)o (b)q(er)g(in)h(the)f(run-)75 214 y(ning)f(homogeneous)f(system\).)29 b(A)18 b(error)g(co)q(de)h(of)e Fl(MPI)p 1062 214 13 2 v 15 w(UNDEFINED)g FC(is)i(allo)o(w)o(ed)g(but)f(strongly)g(dis-)75 271 y(couraged.)31 b(The)19 b(v)m(alue)h(of)e Fl(MPI)p 646 271 V 14 w(MAX)p 752 271 V 15 w(PROCESSOR)p 1008 271 V 13 w(NAME)h FC(is)g(the)g(maxim)o(um)g(length)g(required)h(b)o(y) 75 327 y Fv(MPI)p 160 327 14 2 v 16 w(GET)p 264 327 V 17 w(PROCESSOR)p 540 327 V 18 w(NAME)c FC(\(including,)h(in)f(the)f(C)g (binding,)i(the)e(terminating)h(n)o(ull\).)75 459 y Fv(MPI)p 160 459 V 16 w(GET)p 264 459 V 17 w(HOST\()f(comm,)e(rank\))117 565 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(\(handle\))117 696 y(OUT)108 b Fv(rank)504 b Fq(rank)21 b(of)f Fk(HOST)g Fq(in)h(group)f(of)g Fk(comm)p Fq(;)j Fl(MPI)p 1604 696 13 2 v 15 w(PROCNULL)d Fq(if)905 752 y(there)15 b(is)f(no)g Fk(HOST)f Fq(\(in)o(teger\))75 905 y Fo(int)23 b(MPI)p 245 905 15 2 v 17 w(Get)p 334 905 V 17 w(host\(MPI)p 543 905 V 16 w(Comm)g(comm,)h(int)f(*rank\))75 1020 y(MPI)p 150 1020 V 17 w(GET)p 239 1020 V 17 w(HOST\(COMM,)f(RANK,) h(IERROR\))170 1076 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)166 1191 y FC(This)17 b(routines)g(gets)f(the)g(rank)g(of)g(the)h Fo(HOST)e FC(pro)q(cess)i(in)g(the)g(group)f(asso)q(ciated)g(with)h (comm)o(u-)75 1247 y(nicator)h Fo(comm)p FC(,)g(if)g(there)g(is)h(suc)o (h.)28 b(MPI)18 b(do)q(es)h(not)e(sp)q(ecify)i(what)f(it)g(means)g(for) f(a)h(pro)q(cess)g(to)g(b)q(e)g(a)75 1304 y Fo(HOST)p FC(,)c(nor)h(do)q(es)h(it)f(requires)h(that)f(a)f Fo(HOST)h FC(exists.)75 1586 y Fp(7.1.2)49 b(Bu\013ering)75 1727 y FC(While)15 b(it)g(is)f(p)q(ossible)i(to)d(design)i(algorithms)f (that)f(do)h(not)f(require)i(that)e(an)o(y)h(bu\013ering)h(b)q(e)f(pro) o(vided)75 1783 y(for)e(sending)i(and)e(receiving)i(\\blo)q(c)o(king")g (messages,)e(man)o(y)g(practitioners)h(exp)q(ect)g(that)f(the)g (message-)75 1839 y(passing)k(system)e(will)j(pro)o(vide)f(some)f (bu\013ering.)21 b(That)15 b(is,)g(there)g(is)h(often)f(an)g(exp)q (ectation)h(that)f(the)75 1896 y(program)f(fragmen)o(t,)g(executed)i(b) o(y)f(all)h(tasks,)75 2130 y Fo(MPI_send\()23 b(buf,)g(size,)g (datatype,)g(nbr,)g(tag,)g(comm)h(\))75 2187 y(MPI_recv\()f(buf,)g (size,)g(datatype,)g(nbr,)g(tag,)g(comm)h(\))75 2393 y FC(will)17 b(execute)f(for)f(messages)g(whose)h Fl(size)g FC(is)g(not)f(to)q(o)g(large.)21 b(The)15 b(purp)q(ose)i(of)e(this)h (section)g(is)g(to)f(giv)o(e)75 2450 y(the)j(user)g(a)g(w)o(a)o(y)e(to) i(determine)h(what)e(\\to)q(o)g(large")g(is)i(and)f(to)f(help)i(the)f (implemen)o(tation)h(pro)o(vide)75 2506 y(enough)d(bu\013ering)f(for)g (the)g(user's)g(need.)166 2591 y(Note)f(that)g(nothing)h(in)g(this)g (section)g Fr(r)n(e)n(quir)n(es)i FC(an)e(implemen)o(tation)h(to)e(pro) o(vide)h(an)o(y)f(bu\013ering;)75 2647 y(that)f(is,)h(an)g(implemen)o (tation)h(ma)o(y)f(de\014ne)h(the)f(ab)q(o)o(v)o(e)f(program)g(fragmen) o(t)g(to)g(b)q(e)h(erroneous)g(for)f(an)o(y)75 2704 y(p)q(ositiv)o(e)j (v)m(alue)h(of)d Fl(size)p FC(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 155 163 155 162 bop 75 -100 a Fs(7.1.)34 b(IMPLEMENT)l(A)l(TION)16 b(INF)o(ORMA)l(TION)841 b FC(155)75 45 y Fv(General)16 b(bu\013ering)f(pa)o(rameters)75 178 y(MPI)p 160 178 14 2 v 16 w(GET)p 264 178 V 17 w(BUFFER)p 450 178 V 17 w(P)l(ARAMS\()i(numsend,)g(sizesend,)h(numrecv,)e(sizerecv,)g (totalnumsend,)h(totalsize-)75 235 y(send,)f(totalnumrecv,)f (totalsizerecv\))117 312 y Fq(OUT)108 b Fv(numsend)421 b Fq(Bound)10 b(on)g(the)g(n)o(um)o(b)q(er)f(of)g(p)q(ending)h (messages)g(with)g(source)905 368 y(at)k(calling)e(pro)q(cess)k(and)e (same)f(destination)117 442 y(OUT)108 b Fv(sizesend)436 b Fq(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 498 y(with)f(source)h(at)f(calling)e(pro)q(cess) k(and)e(same)f(destination)117 571 y(OUT)108 b Fv(numrecv)427 b Fq(Bound)16 b(on)e(the)i(n)o(um)o(b)q(er)e(of)g(p)q(ending)i (messages)f(with)f(des-)905 628 y(tination)f(at)h(calling)e(pro)q(cess) k(and)e(same)f(source)117 701 y(OUT)108 b Fv(sizerecv)442 b Fq(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 758 y(with)f(destination)f(at)h(calling)f(pro)q (cess)i(and)f(same)f(source)117 831 y(OUT)108 b Fv(totalnumsend)333 b Fq(Bound)10 b(on)g(the)g(n)o(um)o(b)q(er)f(of)g(p)q(ending)h (messages)g(with)g(source)905 888 y(at)k(the)g(calling)f(pro)q(cess)j (and)d(an)o(y)h(destination)117 961 y(OUT)108 b Fv(totalsizesend)348 b Fq(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 1017 y(with)d(source)i(at)f(the)g(calling)e(pro) q(cess)k(and)d(an)o(y)g(destination)117 1091 y(OUT)108 b Fv(totalnumrecv)339 b Fq(Bound)16 b(on)f(the)h(n)o(um)o(b)q(er)f(of)g (p)q(ending)g(messages)h(from)d(an)o(y)905 1147 y(source)i(with)f(the)g (calling)f(pro)q(cess)j(as)e(destination)117 1221 y(OUT)108 b Fv(totalsizerecv)354 b Fq(Bound)15 b(on)f(the)h(total)f(size)h(in)f (b)o(ytes)h(of)f(p)q(ending)h(messages)905 1277 y(from)9 b(an)o(y)h(source)i(with)e(destination)g(at)g(the)h(calling)e(pro)q (cess)75 1402 y Fo(int)23 b(MPI)p 245 1402 15 2 v 17 w(Get)p 334 1402 V 17 w(buffer)p 495 1402 V 16 w(params\(int)g (*numsend,)f(int)i(*sizesend,)e(int)i(*numrecv,)393 1458 y(int)g(*sizerecv,)e(int)i(*totalnumsend,)e(int)h(*totalsizesend,)393 1514 y(int)h(*totalnumrecv,)e(int)h(*totalsizerecv\))75 1601 y(MPI)p 150 1601 V 17 w(GET)p 239 1601 V 17 w(BUFFER)p 400 1601 V 16 w(PARAMS\(NUMSEND,)f(SIZESEND,)g(NUMRECV,)h(SIZERECV,)g (TOTALNUMSEDND,)393 1657 y(TOTALSIZESEN,)f(TOTALNUMRECV,)g (TOTALSIZERECV,)g(IERROR\))170 1714 y(INTEGER)h(NUMSEND,)g(SIZESEND,)g (NUMRECV,)g(SIZERECV,)f(TOTALNUMSEDND,)170 1770 y(TOTALSIZESEN,)h (TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)166 1857 y FC(A)e(program)f (where)h(at)g(an)o(y)g(p)q(oin)o(t)g(in)h(time)g(no)f(pro)q(cess)g(has) g(more)g(then)g Fv(numsend)i FC(p)q(ending)75 1913 y(messages)13 b(sen)o(t)g(b)o(y)g(it)h(to)f(the)h(same)f(destination,)h(no)f(more)g (than)h Fv(sizesend)h FC(b)o(ytes)e(of)g(data)g(in)h(p)q(ending)75 1970 y(messages)h(sen)o(t)g(b)o(y)h(it)f(to)g(the)h(same)f (destination,)h(no)f(more)g(than)h Fv(totalnumsend)h FC(p)q(ending)g(messages)75 2026 y(sen)o(t)d(b)o(y)h(it)g(to)e(an)o(y)i (destination,)g(no)f(more)g(than)h Fv(totalsizesend)h FC(b)o(ytes)f(of)f(data)g(in)h(p)q(ending)h(messages)75 2083 y(sen)o(t)g(b)o(y)g(it)h(to)e(an)o(y)h(destination,)h(no)f(more)g (than)g Fv(numrecv)h FC(p)q(ending)h(messages)d(sen)o(t)h(to)g(it)g (from)g(the)75 2139 y(same)k(source,)h(no)e(more)h(than)g Fv(sizerecv)g FC(b)o(ytes)g(in)g(p)q(ending)i(messages)e(sen)o(t)f(to)h (it)g(from)f(the)h(same)75 2195 y(source,)g(no)f(more)g(than)g Fv(totalnumrecv)g FC(p)q(ending)i(messages)e(sen)o(t)g(to)g(it)g(from)g (an)o(y)g(source,)h(and)f(no)75 2252 y(more)c(than)g Fv(totalsizerecv)h FC(b)o(ytes)f(in)h(p)q(ending)h(messages)e(sen)o(t)g (to)f(it)i(from)e(an)o(y)h(source)h(is)f(guaran)o(teed)75 2308 y(not)g(to)f(deadlo)q(c)o(k)i(b)q(ecause)h(of)d(the)i(lac)o(k)f (of)g(bu\013er)g(space)h(for)e(p)q(ending)j(messages.)166 2365 y(Examples:)166 2421 y(1.)30 b(Supp)q(ose)20 b(that)e(p)q(ending)i (messages)f(are)f(alw)o(a)o(ys)g(bu\013ered)h(at)f(the)h(sender,)h(all) g(bu\013ers)e(are)75 2478 y(shared,)d(total)f(bu\013er)h(size)h(at)f (the)g(pro)q(cess)g(is)h Fo(B)p FC(,)e(eac)o(h)h(p)q(ending)i(message)e (tak)o(es)f Fo(h)h FC(b)o(ytes)g(of)g(storage,)75 2534 y(in)j(addition)g(to)e(the)h(storage)f(for)g(its)h(data,)g(and)g(no)g (storage)e(is)j(lost)f(due)g(to)g(fragmen)o(tation.)23 b(Then)75 2591 y(p)q(ossible)14 b(return)e(v)m(alues)h(for)e Fv(MPI)p 662 2591 14 2 v 16 w(GET)p 766 2591 V 17 w(BUFFER)p 952 2591 V 17 w(P)l(ARAMS)i FC(are)f Fo(numsend)23 b(=)g(totalnumsend)g (=)g(k,)75 2647 y(sizesend)g(=)g(totalsizesend)g(=)g(B-kh)p FC(,)11 b(and)f Fo(numrecv)23 b(=)g(totalnumrecv)g(=)g(sizerecv)g(=)h (totalsizerecv)75 2704 y(=)g(INT)p 198 2704 15 2 v 16 w(MAX)p FC(,)15 b(for)f(some)h Fo(0)e Fw(<)g Fo(k)f Fw(<)h Fo(B)p Fw(=)p Fo(h)p FC(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 156 164 156 163 bop 75 -100 a FC(156)532 b Fs(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)166 45 y FC(2.)36 b(Supp)q(ose)22 b(that)e(messages)g(are)h(alw)o(a)o(ys)f (bu\013ered)h(at)f(the)h(receiv)o(er,)h(and)f(that)f(a)g(separate)75 102 y(bu\013er)15 b(with)h Fo(C)e FC(b)o(ytes)h(is)h(allo)q(cated)g (for)e(eac)o(h)i(pair)f(of)g(comm)o(unicating)h(pro)q(cesses;)f(en)o (tries)g(for)g(p)q(end-)75 158 y(ing)j(messages)g(are)f(k)o(ept)h(in)g (a)g(seeparate)f(table,)h(with)h(ro)q(om)e(for)g(total)g(of)g Fo(m)h FC(p)q(ending)h(messages)f(at)75 214 y(eac)o(h)13 b(receiving)i(pro)q(cess;)e(the)g(total)g(n)o(um)o(b)q(er)g(of)g(pro)q (cesses)g(is)h Fo(p)p FC(,)e(and)i(no)f(storage)e(is)j(lost)f(due)g(to) g(frag-)75 271 y(men)o(tation.)23 b(Then)16 b(p)q(ossible)i(return)e(v) m(alues)i(for)d Fv(MPI)p 1035 271 14 2 v 16 w(GET)p 1139 271 V 17 w(BUFFER)p 1325 271 V 17 w(P)l(ARAMS)j FC(are)e Fo(sendcount)22 b(=)75 327 y(totalsendcount)g(=)i(sendsize)e(=)i (totalsendsize)e(=)i(INT)p 1200 327 15 2 v 17 w(MAX,)f(recvcount)g(=)g (totalrecvcount)75 384 y(=)h(m,)f(recsize)g(=)h(C)15 b FC(and)g Fo(totalrecvsize)22 b(=)i(\(p-1\))p FA(\002)p Fo(C)p FC(.)166 442 y(The)19 b(call)g(ma)o(y)f(return)g(the)g(v)m(alue) i Fl(MPI)p 872 442 13 2 v 14 w(UNDEFINED)e FC(in)h(an)o(y)f(argumen)o (t.)29 b(If)18 b(this)h(is)g(the)f(case,)75 498 y(then)e(the)f(MPI)g (implemen)o(tation)h(mak)o(es)f(no)g(guaran)o(tee)g(ab)q(out)g (bu\013ering.)166 556 y(Some)k(implemen)o(tations)h(ma)o(y)f(b)q(e)g (able)h(to)f(adjust)f(their)i(bu\013ering.)32 b(In)20 b(addition,)h(since)f(the)75 612 y(parameters)13 b(returned)i(b)o(y)f Fv(MPI)p 640 612 14 2 v 16 w(GET)p 744 612 V 17 w(BUFFER)p 930 612 V 17 w(P)l(ARAMS)h FC(represen)o(t)f(a)g(guaran)o(tee)g(ab)q (out)g(a)g(partic-)75 669 y(ular)g(c)o(hoice)h(of)e(n)o(um)o(b)q(er)h (of)g(bu\013ers,)f(it)i(ma)o(y)e(not)g(guaran)o(tee)g(bu\013ering)i (that)e(it)h(can)g(in)g(fact)g(p)q(erform.)75 725 y(Th)o(us,)k(in)h (example)g(1)f(ab)q(o)o(v)o(e,)g(the)g(implemen)o(tation)h(pic)o(ks)g (an)f(arbitrary)f(v)m(alue)i(for)f Fo(k)p FC(,)g(trading)g(o\013)75 782 y(n)o(um)o(b)q(er)h(of)f(messages)h(for)f(total)g(message)g(size,)j (rather)d(than)g(letting)i(the)f(user)g(mak)o(e)f(this)h(trade-)75 838 y(o\013.)k(In)18 b(order)e(to)g(allo)o(w)h(the)g(MPI)f(user)h(to)f (inquire)j(ab)q(out)d(a)g(sp)q(eci\014c)j(bu\013ering)e(need,)h(the)e (routine)75 895 y Fv(MPI)p 160 895 V 16 w(SUGGEST)p 375 895 V 18 w(BUFFER)p 562 895 V 17 w(P)l(ARAMS)g FC(is)g(pro)o(vided.)75 999 y Fv(MPI)p 160 999 V 16 w(SUGGEST)p 375 999 V 18 w(BUFFER)p 562 999 V 17 w(P)l(ARAMS\()h(numsend,)i(sizesend,)f (numrecv,)f(sizerecv,)g(totalnumsend,)i(to-)75 1056 y(talsizesend,)e (totalnumrecv,)e(totalsizerecv,)g(\015ag\))117 1134 y Fq(IN)155 b Fv(numsend)421 b Fq(Bound)13 b(on)f(n)o(um)o(b)q(er)g(of)f (p)q(ending)i(messages)f(with)g(source)i(at)905 1191 y(calling)f(pro)q(cess)i(and)f(same)f(destination)117 1269 y(IN)155 b Fv(sizesend)436 b Fq(Bound)12 b(on)g(total)f(size)i(in) e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1325 y(source)j(at)f(calling)f(pro)q(cess)i(and)f(same)f(destination)117 1403 y(IN)155 b Fv(numrecv)427 b Fq(Bound)16 b(on)e(the)i(n)o(um)o(b)q (er)e(of)g(p)q(ending)i(messages)f(with)f(des-)905 1459 y(tination)f(at)h(calling)e(pro)q(cess)k(and)e(same)f(source)117 1537 y(IN)155 b Fv(sizerecv)442 b Fq(Bound)12 b(on)g(total)f(size)i(in) e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1593 y(destination)i(at)g(calling)e(pro)q(cess)k(and)e(same)f(source)117 1671 y(IN)155 b Fv(totalnumsend)333 b Fq(Bound)10 b(on)g(the)g(n)o(um)o (b)q(er)f(of)g(p)q(ending)h(messages)g(with)g(source)905 1728 y(at)k(the)g(calling)f(pro)q(cess)j(and)d(an)o(y)h(destination)117 1805 y(IN)155 b Fv(totalsizesend)348 b Fq(Bound)12 b(on)g(total)f(size) i(in)e(b)o(ytes)i(of)e(p)q(ending)h(messages)g(with)905 1862 y(source)j(at)f(the)h(calling)d(pro)q(cess)k(and)e(an)o(y)f (destination)117 1939 y(IN)155 b Fv(totalnumrecv)339 b Fq(Bound)16 b(on)f(the)h(n)o(um)o(b)q(er)f(of)g(p)q(ending)g (messages)h(from)d(an)o(y)905 1996 y(source)i(with)f(the)g(calling)f (pro)q(cess)j(as)e(destination)117 2074 y(IN)155 b Fv(totalsizerecv)354 b Fq(Bound)15 b(on)f(the)h(total)f(size)h(in)f(b)o(ytes)h(of)f(p)q (ending)h(messages)905 2130 y(from)9 b(an)o(y)h(source)i(with)e (destination)g(at)g(the)h(calling)e(pro)q(cess)117 2208 y(OUT)108 b Fv(\015ag)518 b Fl(TRUE)17 b Fq(if)f(the)h(suggestion)h(w)o (as)f(accepted,)i Fl(F)m(ALSE)p Fq(,)c(other-)905 2264 y(wise)75 2390 y Fo(int)23 b(MPI)p 245 2390 15 2 v 17 w(Suggest)p 430 2390 V 16 w(buffer)p 590 2390 V 17 w(params\(int)f (numsend,)h(int)h(sizesend,)e(int)i(numrecv,)393 2446 y(int)g(sizerecv,)e(int)i(totalnumsend,)e(int)h(totalsizesen,)393 2503 y(int)h(totalnumrecv,)e(int)h(totalsizerecv,)f(int)i(*flag\))75 2591 y(MPI)p 150 2591 V 17 w(SUGGEST)p 335 2591 V 16 w(BUFFER)p 495 2591 V 16 w(PARAMS\(NUMSEND,)e(SIZESEND,)h(NUMRECV,)g (SIZERECV,)393 2647 y(TOTALNUMSEND,)f(TOTALSIZESEN,)g(TOTALNUMRECV,)g (TOTALSIZERECV,)g(FLAG,)393 2704 y(IERROR\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 157 165 157 164 bop 75 -100 a Fs(7.1.)34 b(IMPLEMENT)l(A)l(TION)16 b(INF)o(ORMA)l(TION)841 b FC(157)170 45 y Fo(INTEGER)23 b(NUMSEND,)g(SIZESEND,)g(NUMRECV,)g(SIZERECV,)f(TOTALNUMSEND,)170 102 y(TOTALSIZESEN,)h(TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)170 158 y(LOGICAL)h(FLAG)166 246 y FC(If)17 b(a)f(v)m(alue)h(of)f(true)h (is)g(returned)f(in)i Fv(\015ag)e FC(then)h(it)f(is)h(as)f(if)h(the)g (v)m(alues)g(pro)o(vided)h(b)o(y)e(the)h(user)f(in)75 302 y Fv(MPI)p 160 302 14 2 v 16 w(SUGGEST)p 375 302 V 18 w(BUFFER)p 562 302 V 17 w(P)l(ARAMS)f FC(w)o(ere)f(returned)h(b)o (y)f(a)g(call)h(to)e Fv(MPI)p 1378 302 V 16 w(GET)p 1482 302 V 17 w(BUFFER)p 1668 302 V 17 w(P)l(ARAMS)p FC(:)75 359 y(the)i(user)h(is)g(guaran)o(teed)e(that)h(no)g(deadlo)q(c)o(k)i (will)g(o)q(ccur)e(due)h(to)f(lac)o(k)g(of)g(bu\013ers,)g(pro)o(vided)h (it)g(nev)o(er)75 415 y(exceeds)11 b(the)g(limitations)g(on)g(n)o(um)o (b)q(er)f(and)h(size)g(of)f(p)q(ending)i(messages)e(implied)i(b)o(y)f (these)f(parameters.)166 473 y(This)19 b(routines)g(allo)o(ws)f(the)h (user)f(to)g(tell)i(the)e(MPI)g(implemen)o(tation)i(that)e(a)g(certain) h(amoun)o(t)75 530 y(of)c(bu\013ering)i(w)o(ould)f(b)q(e)h(desireable,) g(The)f(MPI)g(implemen)o(tation)h(is)g(not)e(required)i(to)e(pro)o (vide)i(suc)o(h)75 586 y(bu\013ering.)166 644 y(It)e(is)h(exp)q(ected)g (that)f(a)g(common)f(use)i(of)f(these)g(routines)h(will)h(b)q(e)75 745 y Fo(MPI_Get_buffer_params\()k(&sendnum,)h(&sendsize,)h(...)g(\);) 75 802 y(if)h(\(sendsize)e(<)i(buffer_needed\))e({)170 858 y(fprintf\()h(stderr,)g("Program)g(requires)g(more)g(buffering)g (than)g(is)h(available\\n")e(\);)170 915 y(MPI_Abort\()h(1)h(\);)170 971 y(})75 1028 y(else)170 1084 y(exploit)f(buffering)75 1184 y FC(This)d(allo)o(ws)f(a)g(program)f(to)h(fail)h(rather)f(than)g (deadlo)q(c)o(k.)33 b(It)19 b(is)h(also)f(p)q(ossible)i(for)e(a)f (program)h(to)75 1240 y(switc)o(h)c(to)g(a)g(di\013eren)o(t)g (algorithm)g(that)g(requires)h(less)g(or)e(no)i(bu\013ering.)166 1298 y(Another)j(use)g(is)h(to)f(inform)g(the)g(MPI)g(implemen)o (tation)i(of)d(the)i(bu\013er)f(space)g(desires)h(of)f(the)75 1355 y(application:)75 1456 y Fo(MPI_Suggest_buffer_params\()h(1,)k (buffer_needed,)e(1,)h(buffer_needed,)f(0,)i(0,)g(0,)242 1512 y(0,)g(&flag)f(\);)75 1569 y(if)h(\(flag\))f({)170 1625 y(exploit)g(buffering)170 1682 y(})75 1738 y(else)g({)170 1795 y(fprintf\()g(stderr,)g("Program)g(requires)g(more)g(buffering)g (than)g(is)h(available\\n")e(\);)170 1851 y(MPI_abort\()h(1)h(\);)170 1907 y(})75 2036 y Fv(Application)17 b(p)o(rovided)e(bu\013ering)75 2125 y FC(An)k(application)h(program)e(can)h(optionally)h(pro)o(vide)f (space)g(for)f(MPI's)g(use)i(in)f(bu\013ering)g(message)75 2181 y(data)c(via)g(the)g(follo)o(wing)h(call:)75 2286 y Fv(MPI)p 160 2286 V 16 w(USER)p 288 2286 V 17 w(SPECIFIES)p 519 2286 V 17 w(BUFFER\()f(comm,)e(bu\013er,)i(size\))117 2365 y Fq(IN)155 b Fv(comm)470 b Fq(is)16 b(a)f(comm)o(unicator)e(with) j(whic)o(h)f(the)i(bu\013er)f(is)g(to)g(b)q(e)g(as-)905 2421 y(so)q(ciated)f(\(handle\))117 2500 y(IN)155 b Fv(bu\013er)478 b Fq(initial)12 b(bu\013er)j(address)g(\(c)o(hoice\))117 2578 y(IN)155 b Fv(size)519 b Fq(bu\013er)15 b(size,)f(in)g(b)o(ytes)g (\(in)o(teger\))75 2704 y Fo(int)23 b(MPI)p 245 2704 15 2 v 17 w(User)p 358 2704 V 17 w(specifies)p 591 2704 V 16 w(buffer\(MPI)p 847 2704 V 16 w(Comm)g(comm,)g(void*)g(buffer,)g (int)h(size\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 158 166 158 165 bop 75 -100 a FC(158)532 b Fs(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(USER)p 263 45 V 16 w(SPECIFIES)p 495 45 V 16 w(BUFFER\(COMM,)23 b(BUFFER,)f(SIZE,)i(IERROR\))170 102 y()f(BUFFER\(*\))170 158 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR) 166 246 y FC(If)17 b Fv(MPI)p 298 246 14 2 v 16 w(USER)p 426 246 V 17 w(SPECIFIES)p 657 246 V 17 w(BUFFER)g FC(is)h(called)g (and)g(the)f(v)m(alue)h(returned)f(in)h Fo(flag)e FC(is)i Fo(MPI)p 1801 246 15 2 v 17 w(OK)p FC(,)75 302 y(then)j(for)g (subsequen)o(t)g(regular)g(sends)h(using)f(the)g(sp)q(eci\014ed)i(comm) o(unicator,)f(MPI)f(m)o(ust)f(pro)o(vide)75 359 y(as)e(m)o(uc)o(h)g (safet)o(y)f Fr(as)i(if)27 b FC(outgoing)18 b(message)f(data)h(w)o(ere) g(bu\013ered)g(b)o(y)g(the)g(sending)h(pro)q(cess,)g(in)g(the)75 415 y(sp)q(eci\014ed)k(bu\013er)e(space,)h(using)f(a)f(circular)i(con)o (tiguous-space)g(allo)q(cation)f(p)q(olicy)l(.)39 b(The)21 b(bu\013er)f(is)75 472 y(managed)15 b(b)o(y)g(MPI.)166 529 y(Access)k(to)f(the)h(pro)o(vided)h(bu\013er)f(space)g(is)g(not)f (inherited)j(b)o(y)e(comm)o(unicators)f(deriv)o(ed)i(from)75 586 y(the)15 b(initially)j(sp)q(eci\014ed)g(one.)166 720 y Fn(Implemen)o(tati)o(on)12 b(note:)37 b Fq(One)15 b(approac)o(h)f(is)g(for)g(MPI)g(to)g(simply)e(transform)h(all)g(blo)q (c)o(king)g(sends)i(on)75 769 y(the)f(sp)q(eci\014ed)i(comm)o(unicator) 11 b(in)o(to)145 848 y Fc(\017)23 b Fq(cop)o(y)13 b(data)h(to)g (bu\013er)h(space)145 928 y Fc(\017)23 b Fq(issue)14 b(non-blo)q(c)o(king)f(send)i(from)d(the)i(bu\013er)h(cop)o(y)145 1008 y Fc(\017)23 b Fq(return)15 b(to)e(application)145 1088 y Fc(\017)23 b Fq(c)o(hec)o(k)14 b(completion)f(on)g(subsequen)o (t)j(MPI)e(call\(s\))166 1167 y(No)i(doubt)g(man)o(y)e(optimizations)g (within)h(MPI)h(are)h(p)q(ossible|the)f(prop)q(osal)g(just)g(requires)i (that)e(MPI)75 1217 y(pro)o(vide)e(at)f(least)i(this)e(m)o(uc)o(h)g (safet)o(y)m(.)166 1274 y(The)18 b(reason)h(that)f(MPI)h(pro)o(vides)f (the)g(bu\013er)i(rather)e(than)h(the)f(user)h(is)f(that)g(once)h(the)g (user)g(frees)g(a)75 1331 y(comm)o(unicator,)9 b(only)j(MPI)g(kno)o(ws) g(when)h(all)e(data-structures)j(asso)q(ciated)f(with)e(that)i(comm)o (unicator)c(can)j(b)q(e)75 1387 y(freed.)166 1528 y FC(Note)f(that)g (collectiv)o(e)j(comm)o(unications)e(and)g(other)f(nonlo)q(cal)i(op)q (erations)f(will)i(share)d(suggested)75 1584 y(or)k(user-supplied)j (comm)o(unication)d(bu\013ers)h(with)f(p)q(oin)o(t)h(to)e(p)q(oin)o(t)i (comm)o(unication.)75 1735 y Ft(7.2)59 b(Erro)n(r)21 b(handling)75 1839 y FC(This)15 b(section)h(con)o(tains)f(the)g (suggested)g(POSIX)g(binding)i(for)e(error)f(handling)i(in)g(MPI.)e (MPI)h(imple-)75 1896 y(men)o(tations)d(that)h(do)f(not)h(run)g(in)g(a) g(POSIX-complian)o(t)i(en)o(vironmen)o(t)e(ma)o(y)f(need)h(to)g(use)g (alternativ)o(e)75 1952 y(mec)o(hanisms.)166 2010 y(An)k(MPI)h (implemen)o(tation)g(cannot)f(or)g(ma)o(y)f(c)o(ho)q(ose)i(not)f(to)f (handle)j(some)e(errors)f(that)h(o)q(ccur)75 2066 y(during)23 b(MPI)f(calls.)41 b(These)22 b(can)g(include)i(errors)d(that)h (generate)f(exceptions)i(or)f(traps,)g(suc)o(h)g(as)75 2123 y(\015oating)15 b(p)q(oin)o(t)h(errors)f(or)g(access)g (violations.)22 b(The)15 b(set)g(of)g(errors)g(that)g(are)g(not)g (handled)i(b)o(y)e(MPI)g(is)75 2179 y(implemen)o(tation-dep)q(endent.) 166 2237 y(There)d(are)g(three)h(p)q(ossible)h(error)e(handling)i(mo)q (des)e(for)g(those)g(errors)f(that)h(MPI)g(handles,)i(whic)o(h)75 2293 y(are)h(iden)o(ti\014ed)i(b)o(y)e(three)h(constan)o(t)e(v)m (alues:)75 2392 y Fl(MPI)p 152 2392 13 2 v 14 w(ERRORSF)m(A)m(T)m(AL)20 b FC(The)c(error)e(is)i Fr(fatal)f FC(and)h(causes)f(the)g(program)f (to)h(ab)q(ort.)75 2492 y Fl(MPI)p 152 2492 V 14 w(ERRORSRETURN)21 b FC(The)13 b(error)e(causes)i(the)f(MPI)g(routine)h(where)f(it)h(o)q (ccurred)g(to)e(return)i(an)f(error)189 2548 y(co)q(de.)75 2647 y Fl(MPI)p 152 2647 V 14 w(ERRORSCALLUSERROUTINE)20 b FC(The)11 b(error)f(causes)h(MPI)f(to)g(call)i(a)e(user-sp)q (eci\014ed)j(error)d(handling)189 2704 y(routine.)20 b(Suc)o(h)c(a)f(handler)h(can)f(b)q(e)h(sp)q(eci\014ed)i(b)o(y)d(a)g (call)h(to)e Fv(MPI)p 1328 2704 14 2 v 16 w(SET)p 1427 2704 V 17 w(HANDLER)p FC(.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 159 167 159 166 bop 75 -100 a Fs(7.2.)34 b(ERR)o(OR)17 b(HANDLING)1194 b FC(159)166 45 y(A)13 b(user)h(can)f(asso)q(ciate)g(an)g(error)g (handling)i(mo)q(de)e(with)h(a)f(comm)o(unicator.)19 b(The)13 b(sp)q(eci\014ed)j(error)75 102 y(handling)j(mo)q(de)f(will)i (b)q(e)e(used)g(for)g(an)o(y)f(error)g(that)g(o)q(ccurs)h(during)h(a)e (call)i(to)e(MPI)h(for)f(a)g(comm)o(u-)75 158 y(nication)g(with)f(this) g(comm)o(unicator.)21 b(MPI)16 b(calls)h(that)e(are)g(not)g(related)h (to)g(an)o(y)f(comm)o(unicator)g(are)75 214 y(considered)i(to)d(b)q(e)i (attac)o(hed)f(to)f(the)h(comm)o(unicator)g Fl(MPI)p 1108 214 13 2 v 15 w(COMM)p 1253 214 V 14 w(W)o(ORLD)p FC(.)166 271 y(A)k(user)g(ma)o(y)f(also)g(attac)o(h)g(an)h(error)f (handling)j(routine)e(to)f(a)g(comm)o(unicator;)i(This)g(attac)o(he-)75 327 y(men)o(t)f(is)g(purely)h(lo)q(cal:)29 b(di\013eren)o(t)19 b(pro)q(cesses)h(ma)o(y)e(attac)o(h)g(di\013eren)o(t)h(error)g (handlers)h(to)e(the)h(same)75 384 y(comm)o(unicator.)166 440 y(A)13 b(newly)h(created)g(comm)o(unicator)f(inherits)i(the)e (error)g(mo)q(de)g(and)h(the)f(error)g(handling)i(routine,)75 497 y(if)22 b(an)o(y)l(,)g(that)f(is)h(asso)q(ciated)g(with)f(the)h (\\paren)o(t")e(comm)o(unicator.)38 b(In)23 b(particular,)g(the)e(user) h(can)75 553 y(sp)q(ecify)14 b(a)e(\\global")h(error-handling)h(mo)q (de)f(for)f(all)i(comm)o(unicators)e(b)o(y)g(asso)q(ciating)i(this)f (mo)q(de)g(with)75 610 y(the)i(comm)o(unicator)g Fl(MPI)p 527 610 V 14 w(COMM)p 671 610 V 15 w(W)o(ORLD)g FC(immediatly)i(after)d (initialization.)166 666 y(The)h(default)h(error)f(handling)h(mo)q(de)g (for)f Fl(MPI)p 980 666 V 14 w(COMM)p 1124 666 V 15 w(W)o(ORLD)g FC(is)g Fl(MPI)p 1422 666 V 15 w(ERRORSF)m(A)m(T)m(A)o(L)p FC(.)75 770 y Fv(MPI)p 160 770 14 2 v 16 w(GET)p 264 770 V 17 w(ERRORMODE)i(\()d(comm,)f(mo)q(de)i(\))117 847 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(to)j(get)g(error)h(mo)q(de)e(from)f(\(handle\))117 921 y(OUT)108 b Fv(mo)q(de)483 b Fq(curren)o(t)16 b(error)e(handling)f (mo)q(de)g(\(state\))75 1045 y Fo(int)23 b(MPI)p 245 1045 15 2 v 17 w(Get)p 334 1045 V 17 w(errormode\(MPI)p 663 1045 V 15 w(Comm)h(comm,)f(MPI)p 1013 1045 V 17 w(Errmode)g (*mode\))75 1132 y(MPI)p 150 1132 V 17 w(GET)p 239 1132 V 17 w(ERRORMODE\(COMM,)e(MODE,)j(IERROR\))170 1188 y(INTEGER)f(COMM,)h (MODE,)f(IERROR)166 1275 y FC(Gets)15 b(the)g(error)f(handling)j(mo)q (de.)75 1379 y Fv(MPI)p 160 1379 14 2 v 16 w(SET)p 259 1379 V 17 w(ERRORMODE)g(\()d(comm,)f(mo)q(de)i(\))117 1456 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(to)j(set)h(error)g(mo)q(de)e(for)g(\(handle\))117 1530 y(IN)155 b Fv(mo)q(de)483 b Fq(new)15 b(error)f(mo)q(de)f(\(state\))75 1654 y Fo(int)23 b(MPI)p 245 1654 15 2 v 17 w(Set)p 334 1654 V 17 w(errormode\(MPI)p 663 1654 V 15 w(Comm)h(comm,)f(MPI)p 1013 1654 V 17 w(Errmode)g(mode\))75 1741 y(MPI)p 150 1741 V 17 w(SET)p 239 1741 V 17 w(ERRORMODE\(COMM,)e(MODE,)j(IERROR\)) 170 1797 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)166 1884 y FC(Sets)15 b(the)g(error)g(handling)i(mo)q(de.)75 1987 y Fv(MPI)p 160 1987 14 2 v 16 w(SET)p 259 1987 V 17 w(ERRHANDLER\()f (comm,)d(handler)j(\))117 2064 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(to)j(set)h(the)f(error)h(handler)f(for)g (\(handle\))117 2138 y(IN)155 b Fv(handler)449 b Fq(user-de\014ned)16 b(error)f(handler)f(to)g(call)75 2263 y Fo(int)23 b(MPI)p 245 2263 15 2 v 17 w(Set)p 334 2263 V 17 w(errhandler\(MPI)p 687 2263 V 15 w(Comm)h(comm,)f(void)g(handler\))75 2349 y(MPI)p 150 2349 V 17 w(SET)p 239 2349 V 17 w(ERRHANDLER\(COMM,)e (HANDLER,)i(IERROR\))170 2406 y(EXTERNAL)g(HANDLER)170 2462 y(INTEGER)g(COMM,)h(IERROR)166 2549 y FC(This)16 b(routine)f(returns)g(the)h(previous)g(error)e(handler)i(as)f(its)h(v)m (alue.)166 2605 y(An)f(error)g(handler)h(is)g(a)f(C)g(function)h (declared)g(as)75 2704 y Fo(typedef)23 b(void)g (\(MPI_ERRHANDLER\)\(MPICOMM,)e(int)i(*,)h(...\);)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 160 168 160 167 bop 75 -100 a FC(160)532 b Fs(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y FC(The)d(\014rst)g(argumen)o(t)g(is)h(the)f(comm)o(unicator)g(in)h (use.)20 b(The)13 b(second)h(is)g(a)e(p)q(oin)o(ter)i(to)f(the)g(error) g(co)q(de)h(to)75 102 y(b)q(e)f(returned)f(b)o(y)g(the)g(MPI)g (routine.)19 b(The)12 b(remaining)h(argumen)o(ts)e(are)h(\\)p Fo(varargs)p FC(")e(argumen)o(ts)h(whose)75 158 y(n)o(um)o(b)q(er)j (and)f(meaning)h(is)g(implemen)o(tation-dep)q(enden)o(t.)22 b(An)13 b(implemen)o(tation)i(should)f(clearly)g(do)q(c-)75 214 y(umen)o(t)h(these)h(argumen)o(ts.)166 273 y(There)f(are)f(t)o(w)o (o)g(prede\014ned)i(error)e(handlers)i(that)e(the)h(user)f(ma)o(y)h (use)g(instead)g(of)f(writing)h(their)75 329 y(o)o(wn:)75 430 y Fl(MPI)p 152 430 13 2 v 14 w(DEF)m(A)o(UL)m(THANDLER)22 b FC(ab)q(orts)14 b(from)h(within)h(MPI.)75 532 y Fl(MPI)p 152 532 V 14 w(NOHANDLER)24 b FC(returns)15 b(the)g(error)f(v)m(alue)j (and)e(has)g(no)g(further)g(e\013ect.)75 680 y Fv(MPI)p 160 680 14 2 v 16 w(ERROR)p 323 680 V 18 w(STRING\()g(erro)o(rco)q(de,) f(len,)i(string)f(\))117 759 y Fq(IN)155 b Fv(erro)o(rco)q(de)410 b Fq(Error)15 b(co)q(de)g(returned)g(b)o(y)f(an)f(MPI)i(routine)117 838 y(IN)155 b Fv(len)533 b Fq(Length)14 b(of)g Fl(string)117 917 y Fq(OUT)108 b Fv(string)481 b Fq(T)m(ext)14 b(that)g(corresp)q (onds)i(to)e(the)g Fl(erro)o(rco)q(de)75 1043 y Fo(int)23 b(MPI)p 245 1043 15 2 v 17 w(Error)p 382 1043 V 17 w(string\(int)f (errorcode,)h(int)g(len,)g(char)h(*string\))75 1132 y(MPI)p 150 1132 V 17 w(ERROR)p 287 1132 V 16 w(STRING\(ERRORCODE,)e(STRING,)h (IERROR\))170 1188 y(INTEGER)g(ERRORCODE,)g(IERROR)170 1245 y(CHARACTER*\(*\))g(STRING)75 1428 y Ft(7.3)59 b(Sta)n(rtup)75 1533 y FC(One)13 b(goal)g(of)f(MPI)h(is)g(to)f(ac)o(hiev)o(e)h Fr(sour)n(c)n(e)h(c)n(o)n(de)g(p)n(ortability)p FC(.)19 b(By)13 b(this)g(w)o(e)f(mean)h(that)f(a)g(program)g(writ-)75 1590 y(ten)j(using)i(MPI)e(and)g(complying)i(with)f(the)f(relev)m(an)o (t)h(language)g(standards)f(is)g(p)q(ortable)h(as)f(written,)75 1646 y(and)k(m)o(ust)e(not)h(require)i(an)o(y)e(source)g(co)q(de)h(c)o (hanges)f(when)h(mo)o(v)o(ed)f(from)g(one)g(system)g(to)g(another.)75 1703 y(This)h(explicitly)h(do)q(es)f Fr(not)j FC(sa)o(y)17 b(an)o(ything)i(ab)q(out)f(ho)o(w)f(an)h(MPI)g(program)f(is)i(started)e (or)g(launc)o(hed)75 1759 y(from)h(the)i(command)f(line,)i(nor)e(what)f (the)i(user)f(m)o(ust)f(do)i(to)e(set)h(up)h(the)f(en)o(vironmen)o(t)g (in)h(whic)o(h)75 1816 y(an)h(MPI)g(program)f(will)j(run.)38 b(Ho)o(w)o(ev)o(er,)21 b(an)h(implemen)o(tation)g(ma)o(y)e(require)i (some)f(setup)h(to)e(b)q(e)75 1872 y(p)q(erformed)c(b)q(efore)h(other)f (MPI)g(routines)g(ma)o(y)g(b)q(e)h(called.)24 b(T)l(o)16 b(pro)o(vide)g(for)g(this,)g(MPI)h(includes)h(an)75 1929 y(initialization)g(routine)e Fv(MPI)p 582 1929 14 2 v 16 w(INIT)p FC(.)75 2034 y Fv(MPI)p 160 2034 V 16 w(INIT\(\))75 2140 y Fo(int)23 b(MPI)p 245 2140 15 2 v 17 w(Init\(int)g(*argc,)g (char)g(**argv\))75 2228 y(MPI)p 150 2228 V 17 w(INIT\(IERROR\))170 2284 y(INTEGER)g(IERROR)166 2373 y FC(This)17 b(routine)g(m)o(ust)f(b)q (e)h(called)h(b)q(efore)f(an)o(y)f(other)g(MPI)g(routine.)24 b(It)17 b(m)o(ust)f(b)q(e)h(called)h(at)e(most)75 2429 y(once;)f(subsequen)o(t)h(calls)g(are)f(erroneous)g(\(see)g Fv(MPI)p 982 2429 14 2 v 16 w(INITIALIZED)p FC(\).)166 2488 y(All)j(MPI)f(programs)e(m)o(ust)i(con)o(tain)g(a)f(call)i(to)e Fv(MPI)p 1092 2488 V 16 w(init)p FC(;)i(this)f(routine)h(m)o(ust)e(b)q (e)h(called)i(b)q(efore)75 2544 y(an)o(y)e(other)h(MPI)f(routine)h (\(apart)f(from)g Fo(MPI)p 877 2544 15 2 v 16 w(INITIALIZED)p FC(\))f(is)i(called.)29 b(The)18 b(v)o(ersion)g(for)f(ANSI)h(C)75 2600 y(accepts)d(the)h Fl(a)o(rgc)f FC(and)g Fl(a)o(rgv)g FC(that)g(are)g(pro)o(vided)h(b)o(y)f(the)g(argumen)o(ts)f(to)h Fo(main)p FC(:)75 2704 y Fo(MPI_init\()23 b(argc,)g(argv)g(\);)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 161 169 161 168 bop 75 -100 a Fs(7.3.)34 b(ST)l(AR)l(TUP)1411 b FC(161)75 45 y(The)15 b(F)l(ortran)f(v)o(ersion)i(tak)o(es)e(no)h (argumen)o(ts.)75 149 y Fv(MPI)p 160 149 14 2 v 16 w(FINALIZE\(\))75 252 y Fo(int)23 b(MPI)p 245 252 15 2 v 17 w(Finalize\(\))75 339 y(MPI)p 150 339 V 17 w(FINALIZE\(IERROR\))170 395 y(INTEGER)g(IERROR)166 482 y FC(This)17 b(routines)g(cleans)g(all)h (MPI)e(state.)23 b(Once)18 b(this)f(routine)g(is)g(called,)h(no)e(MPI)h (routine)g(\(ev)o(en)75 538 y Fv(MPI)p 160 538 14 2 v 16 w(INIT)p FC(\))12 b(ma)o(y)h(b)q(e)h(called.)21 b(The)14 b(user)g(m)o(ust)f(ensure)h(that)f(all)h(p)q(ending)i(comm)o (unications)e(in)o(v)o(olving)75 595 y(a)h(pro)q(cess)g(complete)h(b)q (efore)g(the)f(pro)q(cess)g(calls)i Fv(MPI)p 1023 595 V 16 w(FINALIZE)p FC(.)75 698 y Fv(MPI)p 160 698 V 16 w(INITIALIZED\()c(\015ag)i(\))117 776 y Fq(OUT)108 b Fv(\015ag)518 b Fq(F)p FC(lag)15 b(is)f(true)h(if)f Fv(MPI)p 1270 776 V 16 w(INIT)g FC(has)g(b)q(een)h(called)h(and)f(false)905 832 y(otherwise.)75 957 y Fo(int)23 b(MPI)p 245 957 15 2 v 17 w(Initialized\(int)f(*flag\))75 1043 y(MPI)p 150 1043 V 17 w(INITIALIZED\(FLAG,)f(IERROR\))170 1100 y(LOGICAL)i(FLAG)170 1156 y(INTEGER)g(IERROR)166 1242 y FC(This)c(routine)f(ma)o(y)g(b)q(e)h (used)g(to)e(determine)i(whether)g Fv(MPI)p 1242 1242 14 2 v 16 w(INIT)e FC(has)h(b)q(een)h(called.)31 b(It)18 b(is)h(the)75 1299 y Fr(only)c FC(routine)g(that)g(ma)o(y)f(b)q(e)i (called)h(b)q(efore)e Fv(MPI)p 938 1299 V 16 w(INIT)g FC(is)g(called.)75 1403 y Fv(MPI)p 160 1403 V 16 w(ABORT\()h(comm,)d (erro)o(rco)q(de)h(\))117 1480 y Fq(IN)155 b Fv(comm)470 b Fq(comm)o(unicator)11 b(of)j(tasks)g(to)g(ab)q(ort)117 1555 y(IN)155 b Fv(erro)o(rco)q(de)410 b Fq(error)15 b(co)q(de)g(to)e(return)j(to)d(in)o(v)o(oking)f(en)o(vironmen)o(t)75 1679 y Fo(int)23 b(MPI)p 245 1679 15 2 v 17 w(Abort\(MPI)p 478 1679 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 1766 y(MPI)p 150 1766 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1822 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)166 1909 y FC(This)d(routine)f(mak)o(es)g(a)g(\\b)q(est)g(attempt")f(to)h(ab)q (ort)g(all)h(tasks)e(in)i(the)g(group)e(of)h Fv(comm)p FC(.)27 b(This)75 1965 y(function)21 b(do)q(es)g(not)f(require)h(that)f (the)g(in)o(v)o(oking)h(en)o(vironmen)o(t)f(tak)o(e)g(an)o(y)g(action)h (with)f(the)h(error)75 2022 y(co)q(de.)g(Ho)o(w)o(ev)o(er,)14 b(a)h(Unix)i(or)e(POSIX)h(en)o(vironmen)o(t)g(should)g(handle)h(this)f (as)f(a)g Fo(return)23 b(errorcode)75 2078 y FC(from)14 b(the)i(main)f(program)f(or)h(an)g Fo(abort\(errorcode\))p FC(.)166 2135 y(MPI)f(implemen)o(tations)h(are)f(required)h(to)e (de\014ne)i(the)f(b)q(eha)o(vior)h(of)e Fv(MPI)p 1443 2135 14 2 v 16 w(ABORT)i FC(at)f(least)g(for)f(a)75 2191 y Fo(comm)i FC(of)f Fl(MPI)p 314 2191 13 2 v 15 w(COMM)p 459 2191 V 14 w(W)o(ORLD)p FC(.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 162 170 162 169 bop 75 359 a Fu(Chapter)34 b(8)75 570 y Fy(Pro\014ling)42 b(In)m(terface)75 814 y Ft(8.1)59 b(Requirements)75 921 y FC(T)l(o)15 b(meet)g(the)g(MPI)g(pro\014ling)i(in)o(terface,)e(an)g (implemen)o(tation)i(of)d(the)i(MPI)f(functions)h Fr(must)131 1027 y FC(1.)22 b(pro)o(vide)14 b(a)f(mec)o(hanism)h(through)f(whic)o (h)h(all)g(of)f(the)g(MPI)h(de\014ned)h(functions)f(ma)o(y)e(b)q(e)i (accessed)189 1083 y(with)e(a)g(name)f(shift.)20 b(Th)o(us)11 b(all)i(of)f(the)g(MPI)g(functions)h(\(whic)o(h)f(normally)g(start)f (with)h(the)g(pre\014x)189 1140 y(\\)p Fo(MPI)p 287 1140 15 2 v 16 w FC("\))j(should)h(also)f(b)q(e)h(accessable)g(with)g(the)f (pre\014x)h(\\)p Fo(PMPI)p 1300 1140 V 16 w FC(".)131 1246 y(2.)22 b(ensure)c(that)f(those)h(MPI)g(functions)h(whic)o(h)f (are)g(not)f(replaced)i(ma)o(y)f(still)h(b)q(e)g(link)o(ed)g(in)o(to)f (an)189 1302 y(executable)e(image)f(without)h(causing)f(name)h (clashes.)131 1408 y(3.)22 b(do)q(cumen)o(t)c(the)f(implemen)o(tation)i (of)e(di\013eren)o(t)h(language)g(bindings)h(of)e(the)h(MPI)g(in)o (terface)f(if)189 1464 y(they)g(are)h(la)o(y)o(ered)g(on)f(top)g(of)h (eac)o(h)f(other,)h(so)f(that)g(the)h(pro\014ler)g(dev)o(elop)q(er)h (kno)o(ws)e(whether)189 1521 y(she)d(m)o(ust)f(implemen)o(t)i(the)e (pro\014le)i(in)o(terface)f(for)f(eac)o(h)h(binding,)h(or)e(can)h (economise)h(b)o(y)e(imple-)189 1577 y(men)o(ting)i(it)h(only)f(for)g (the)g(lo)o(w)o(est)g(lev)o(el)h(routines.)75 1737 y Ft(8.2)59 b(Discussion)75 1845 y FC(The)13 b(ob)s(jectiv)o(e)g(of)g (the)g(MPI)g(pro\014ling)h(in)o(terface)g(is)f(to)f(ensure)i(that)e(it) i(is)f(relativ)o(ely)h(easy)f(for)g(authors)75 1901 y(of)18 b(pro\014ling)i(\(and)f(other)f(similar\))h(to)q(ols)g(to)f(in)o (terface)h(their)g(co)q(des)g(to)f(MPI)h(implemen)o(tations)h(on)75 1958 y(di\013eren)o(t)15 b(mac)o(hines.)166 2017 y(Since)20 b(MPI)e(is)h(a)f(mac)o(hine)h(indep)q(enden)o(t)h(standard)e(with)h (man)o(y)f(di\013eren)o(t)g(implemen)o(tations,)75 2074 y(it)e(is)h(unreasonable)g(to)e(exp)q(ect)i(that)f(the)g(authors)f(of)h (pro\014ling)h(to)q(ols)f(for)g(MPI)g(will)i(ha)o(v)o(e)d(access)i(to) 75 2130 y(the)c(source)h(co)q(de)f(whic)o(h)h(implemen)o(ts)h(MPI)e(on) g(an)o(y)g(particular)h(mac)o(hine.)20 b(It)13 b(is)h(therefore)f (necessary)75 2187 y(to)21 b(pro)o(vide)i(a)f(mec)o(hanism)g(b)o(y)g (whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o(h)g(to)q(ols)g(can)g (collect)i(whatev)o(er)75 2243 y(p)q(erformance)15 b(information)h (they)f(wish)h Fr(without)g FC(access)f(to)g(the)g(underlying)i (implemen)o(tation.)166 2302 y(W)l(e)d(b)q(eliev)o(e)i(that)e(ha)o (ving)g(suc)o(h)g(an)h(in)o(terface)f(is)h(imp)q(ortan)o(t)e(if)i(MPI)f (is)g(to)g(b)q(e)h(attractiv)o(e)e(to)g(end)75 2359 y(users,)i(since)i (the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g(di\013eren)o(t)h(to)q (ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h(factor)f(in)h(attracting) 75 2415 y(users)f(to)g(the)g(MPI)g(standard.)166 2475 y(The)e(pro\014ling)h(in)o(terface)e(is)h(just)g(that,)f(an)g(in)o (terface.)20 b(It)12 b(sa)o(ys)g Fr(nothing)g FC(ab)q(out)g(the)h(w)o (a)o(y)f(in)h(whic)o(h)75 2531 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f(to)g(la)o(y)h(do)o(wn)g(what)g (information)g(is)h(collected)g(through)75 2588 y(the)g(in)o(terface,)g (or)g(ho)o(w)g(the)g(collected)i(information)e(is)h(sa)o(v)o(ed,)e (\014ltered,)i(or)f(displa)o(y)o(ed.)166 2647 y(While)g(the)g(initial)h (imp)q(etus)f(for)f(the)g(dev)o(elopmen)o(t)h(of)e(this)i(in)o(terface) f(arose)g(from)f(the)h(desire)i(to)75 2704 y(p)q(ermit)g(the)f (implemen)o(tation)i(of)d(pro\014ling)j(to)q(ols,)d(it)i(is)f(clear)h (that)f(an)g(in)o(terface)g(lik)o(e)h(that)f(sp)q(eci\014ed)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 163 171 163 170 bop 75 -100 a Fs(8.3.)34 b(LOGIC)16 b(OF)f(THE)g(DESIGN)1092 b FC(163)75 45 y(ma)o(y)17 b(also)h(pro)o(v)o(e)f(useful)i(for)e(other) h(purp)q(oses,)g(suc)o(h)g(as)g(\\in)o(ternet)o(w)o(orking")f(m)o (ultiple)i(MPI)f(imple-)75 102 y(men)o(tations.)j(Since)c(all)f(that)f (is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there)h(is)g(no)f(ob)s (jection)h(to)f(its)h(b)q(eing)g(used)75 158 y(wherev)o(er)f(it)h(is)f (useful.)166 221 y(As)j(the)g(issues)h(b)q(eing)g(addressed)f(here)h (are)f(in)o(timately)h(tied)f(up)h(with)f(the)g(w)o(a)o(y)f(in)i(whic)o (h)g(ex-)75 277 y(ecutable)j(images)e(are)g(built,)j(whic)o(h)e(ma)o(y) f(di\013er)h(greatly)f(on)g(di\013eren)o(t)h(mac)o(hines,)h(the)e (examples)75 334 y(giv)o(en)c(b)q(elo)o(w)g(should)h(b)q(e)f(treated)g (solely)g(as)f(one)h(w)o(a)o(y)f(of)g(implemen)o(ting)j(the)d(ob)s (jectiv)o(e)h(of)f(the)h(MPI)75 390 y(pro\014ling)h(in)o(terface.)k (The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o (tation)h(are)e(those)g(detailed)75 447 y(in)i(the)f(Requiremen)o(ts)i (section)e(ab)q(o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this) i(c)o(hapter)f(is)g(only)h(presen)o(t)f(as)75 503 y(justi\014cation)g (and)f(discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.) 166 566 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f (in)h(whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed) 75 623 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 679 y(equally)d(v)m(alid\).)75 860 y Ft(8.3)59 b(Logic)20 b(of)g(the)f(design)75 974 y FC(Pro)o(vided)d(that)f(an)g(MPI)g (implemen)o(tation)i(meets)e(the)h(requiremen)o(ts)g(ab)q(o)o(v)o(e,)e (it)i(is)g(p)q(ossible)h(for)e(the)75 1030 y(implemen)o(tor)20 b(of)e(the)h(pro\014ling)h(system)f(to)f(in)o(tercept)i(all)f(of)g(the) g(MPI)g(calls)h(whic)o(h)f(are)g(made)g(b)o(y)75 1087 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h(collect)g(whatev)o (er)f(information)g(she)h(requires)g(b)q(efore)f(calling)75 1143 y(the)c(underlying)i(MPI)d(implemen)o(tation)i(\(through)f(its)g (name)f(shifted)i(en)o(try)e(p)q(oin)o(ts\))h(to)f(ac)o(hiev)o(e)i(the) 75 1200 y(desired)g(e\013ects.)75 1380 y Ft(8.4)59 b(Examples)75 1496 y Fp(8.4.1)49 b(Pro\014ler)16 b(implementation)75 1595 y FC(Supp)q(ose)i(that)d(the)i(pro\014ler)g(wishes)h(to)e(accum)o (ulate)h(the)f(total)g(n)o(um)o(b)q(er)h(of)f(times)h(the)g(MPI)p 1737 1595 14 2 v 16 w(SEND)75 1651 y(function)22 b(is)g(called,)j (along)c(with)h(the)f(total)g(elapsed)i(time)f(sp)q(en)o(t)f(in)i(the)e (function.)40 b(This)22 b(could)75 1708 y(trivially)17 b(b)q(e)f(ac)o(hiev)o(ed)g(as)f(follo)o(ws.)75 1853 y Fo(static)23 b(int)g(totalSendCount;)75 1909 y(static)g(int)g (totalTime;)75 1966 y(extern)g(int)g(now\(void\);)71 b(/*)23 b(Some)h(suitable)e(timer)i(function)e(!)i(*/)75 2079 y(int)f(MPI_Send\(void*)f(buf,)i(int)f(count,)g(MPI_Datatype)f (datatype,)h(int)h(dest,)385 2135 y(int)g(tag,)f(MPI_Comm)g(comm\))75 2192 y({)147 2248 y(int)g(tstart)g(=)h(now\(\);)147 2305 y(/*)f(Now)h(call)f(the)g(real)h(send,)f(passing)g(along)g(all)g(the)h (arguments)f(*/)147 2361 y(int)g(result)g(=)h(PMPI_Send\(buf,)e(count,) h(datatype,)f(dest,)i(tag,)f(comm\);)147 2474 y(totalSendCount++;)475 b(/*)23 b(Accumulate)g(byte)g(count)g(*/)147 2530 y(totalTime)46 b(+=)24 b(now\(\))f(-)h(tstart;)190 b(/*)23 b(and)h(time)333 b(*/)147 2643 y(return)23 b(result;)75 2700 y(})-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 164 172 164 171 bop 75 -100 a FC(164)873 b Fs(CHAPTER)15 b(8.)35 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)75 45 y Fp(8.4.2)49 b(MPI)17 b(lib)o(ra)o(ry)f(implementation)75 131 y FC(On)i(a)f(Unix)h (system,)f(in)h(whic)o(h)g(the)g(MPI)f(library)h(is)g(implemen)o(ted)h (in)f(C,)f(then)g(there)h(are)e(v)m(arious)75 187 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i(t)o(w)o(o)e(of)g(the)h(most)g(ob)o(vious)g (are)g(presen)o(ted)g(here.)35 b(Whic)o(h)20 b(is)h(b)q(etter)75 244 y(dep)q(ends)c(on)e(whether)g(the)h(link)o(er)g(and)f(compiler)i (supp)q(ort)e(w)o(eak)g(sym)o(b)q(ols.)75 364 y Fv(Systems)h(with)g(w)o (eak)f(symb)q(ols)75 450 y FC(If)k(the)f(compiler)i(and)f(link)o(er)h (supp)q(ort)f(w)o(eak)f(external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g(system)75 506 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h(library)g(is)f(required)h (through)f(the)g(use)g(of)g Fo(#pragma)23 b(weak)13 b FC(th)o(us)75 563 y(:{)75 669 y Fo(#pragma)23 b(weak)g(MPI_EXAMPLE)g(=) g(PMPI_EXAMPLE)75 782 y(int)g(PMPI_EXAMPLE\(/*)f(appropriate)h(args)g (*/\))75 838 y({)170 895 y(/*)h(Useful)f(content)g(*/)75 951 y(})166 1057 y FC(The)d(e\013ect)g(of)g(this)g Fo(#pragma)f FC(is)i(to)e(de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Fo(MPI)p 1472 1057 15 2 v 17 w(EXAMPLE)f FC(as)h(a)g(w)o(eak)75 1114 y(de\014nition,)d(this)f(means)g(that)f(the)g(link)o(er)i(will)g (not)e(complain)i(if)f(there)g(is)g(another)f(de\014nition)j(of)d(the) 75 1170 y(sym)o(b)q(ol)g(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1226 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1346 y Fv(Systems)h(without)h(w)o(eak)e(symb)q(ols) 75 1432 y FC(In)k(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h(then)g (one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h(the)f(C)h (macro)75 1489 y(pre-pro)q(cessor)c(th)o(us)g(:{)75 1595 y Fo(#ifdef)23 b(PROFILELIB)75 1651 y(#)95 b(ifdef)23 b(__STDC__)75 1708 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1764 y(#)95 b(else)75 1821 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 1877 y(#)95 b(endif)75 1934 y(#else)75 1990 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 2047 y(#endif)166 2153 y FC(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2259 y Fo(int)23 b(FUNCTION\(MPI_EXAMPLE\)\(/*)e (appropriate)h(args)i(*/\))75 2315 y({)170 2372 y(/*)g(Useful)f (content)g(*/)75 2428 y(})166 2534 y FC(The)d(same)g(source)g(\014le)h (can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q(duce)i(b)q(oth)f(v)o (ersions)g(of)g(the)g(library)l(,)75 2591 y(dep)q(ending)d(on)f(the)f (state)f(of)h(the)g Fo(PROFILELIB)f FC(macro)h(sym)o(b)q(ol.)166 2647 y(It)e(is)g(required)h(that)e(the)h(standard)f(MPI)h(library)g(b)q (e)h(built)g(in)f(suc)o(h)g(a)g(w)o(a)o(y)e(that)h(the)h(inclusion)i (of)75 2704 y(MPI)g(functions)g(can)g(b)q(e)h(ac)o(hiev)o(ed)f(one)g (at)g(a)f(time.)20 b(This)c(is)f(a)f(somewhat)g(unpleasan)o(t)i (requiremen)o(t,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 165 173 165 172 bop 75 -100 a Fs(8.5.)34 b(MUL)l(TIPLE)16 b(LEVELS)g(OF)f (INTER)o(CEPTION)739 b FC(165)75 45 y(since)22 b(it)f(ma)o(y)f(mean)h (that)f(eac)o(h)g(external)i(function)f(has)g(to)f(b)q(e)h(compiled)i (from)d(a)g(separate)g(\014le.)75 102 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g(necessary)g(so)f(that)g(the)g(author)g(of)g(the)h (pro\014ling)g(library)g(need)h(only)f(de\014ne)g(those)75 158 y(MPI)f(functions)g(whic)o(h)g(she)g(wishes)h(to)e(in)o(tercept,)h (references)g(to)f(an)o(y)g(others)g(b)q(eing)i(ful\014lled)h(b)o(y)e (the)75 214 y(normal)g(MPI)g(library)l(.)21 b(Therefore)15 b(the)h(link)g(step)f(can)h(lo)q(ok)f(something)h(lik)o(e)g(this)g(:{) 75 321 y Fo(\045)24 b(cc)f(...)h(-lmyprof)f(-lpmpi)g(-lmpi)166 427 y FC(Here)e Fo(libmyprof.a)e FC(con)o(tains)i(the)g(pro\014ler)g (functions)h(whic)o(h)f(in)o(tercept)g(some)g(of)f(the)h(MPI)75 483 y(functions.)j Fo(libpmpi.a)15 b FC(con)o(tains)i(the)f(\\name)h (shifted")g(MPI)f(functions,)h(and)g Fo(libmpi.a)e FC(con)o(tains)75 540 y(the)g(normal)g(de\014nitions)i(of)e(the)g(MPI)h(functions.)75 683 y Ft(8.5)59 b(Multiple)20 b(levels)f(of)h(interception)75 785 y FC(The)15 b(sc)o(heme)h(giv)o(en)g(here)f(do)q(es)h(not)f (directly)h(supp)q(ort)f(the)h(nesting)g(of)e(pro\014ling)j(functions,) e(since)i(it)75 841 y(pro)o(vides)f(only)f(a)g(single)h(alternativ)o(e) g(name)f(for)f(eac)o(h)h(MPI)g(function.)21 b(Consideration)16 b(w)o(as)e(giv)o(en)i(to)75 898 y(an)f(implemen)o(tation)i(whic)o(h)f (w)o(ould)g(allo)o(w)g(m)o(ultiple)h(lev)o(els)g(of)e(call)h(in)o (terception,)h(ho)o(w)o(ev)o(er)d(w)o(e)h(w)o(ere)75 954 y(unable)j(to)e(construct)g(an)g(implemen)o(tation)i(of)e(this)h (whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o(wing)i(disadv)m(an-) 75 1010 y(tages)143 1104 y FA(\017)23 b FC(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 1198 y FA(\017)23 b FC(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o (en)g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 1292 y(Since)i(one)e(of)g(the)h(ob)s(jectiv)o(es)f(of)g(MPI)h(is)f(to)g (p)q(ermit)h(e\016cien)o(t,)g(lo)o(w)f(latency)h(implemen)o(tations,)h (and)75 1348 y(it)j(is)f(not)g(the)h(business)g(of)f(a)g(standard)g(to) g(require)h(a)f(particular)h(implemen)o(tation)g(language,)g(w)o(e)75 1405 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h(outlined)h(ab)q(o)o(v) o(e.)166 1461 y(Note,)f(ho)o(w)o(ev)o(er,)f(that)h(it)h(is)f(p)q (ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o(v)o(e)g(to)g(implemen) o(t)i(a)e(m)o(ulti-lev)o(el)75 1518 y(system,)k(since)g(the)g(function) g(called)i(b)o(y)d(the)h(user)f(ma)o(y)g(call)i(man)o(y)e(di\013eren)o (t)g(pro\014ling)i(functions)75 1574 y(b)q(efore)15 b(calling)i(the)f (underlying)h(MPI)e(function.)166 1631 y(Unfortunately)22 b(suc)o(h)g(an)g(implemen)o(tation)h(ma)o(y)e(require)h(more)g(co-op)q (eration)g(b)q(et)o(w)o(een)g(the)75 1687 y(di\013eren)o(t)g (pro\014ling)i(libraries)g(than)e(is)h(required)g(for)e(the)i(single)g (lev)o(el)h(implemen)o(tation)f(detailed)75 1744 y(ab)q(o)o(v)o(e.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 166 174 166 173 bop 75 359 a Fu(Chapter)34 b(9)75 569 y Fy(Initial)43 b(Implemen)m(tation)f(Subset)75 812 y Ft(9.1)59 b(Intro)r(duction)75 919 y FC(This)17 b(c)o(hapter)f(de\014nes)h(the)g(minimal)h(subset)e (of)g(MPI)g(for)g(initial)i(implemen)o(tation.)25 b(This)17 b(subset)f(is)75 975 y(b)q(eing)c(de\014ned)h(so)d(that)g(consisten)o (t)i(implemen)o(tations)g(can)f(app)q(ear)g(more)g(rapidly)l(.)20 b(It)11 b(w)o(as)f(recognized)75 1032 y(early)17 b(in)h(the)f(pro)q (cess)g(that)f(MPI)h(needed)h(to)e(app)q(ear)h(as)f(quic)o(kly)i(as)f (p)q(ossible)h(and)f(practical.)26 b(The)75 1088 y(creation)16 b(of)f(a)g(subset)h(will)h(hop)q(efully)g(allo)o(w)f(users)g(earlier)g (access)g(to)e(the)i(standard)f(and)h(still)h(allo)o(w)75 1145 y(for)e(the)g(writing)h(of)e(p)q(ortable)i(message)f(passing)g(co) q(des.)166 1204 y(This)20 b(subset)g(should)h(not)e(in)i(an)o(y)e(w)o (a)o(y)g(b)q(e)h(construed)g(as)f(a)h(limitation)h(on)e(MPI)h (implemen-)75 1260 y(tations.)35 b(It)20 b(is)h(strictly)g(the)g(minim) o(um)g(necessary)g(to)e(ha)o(v)o(e)h(an)h(initial)h(MPI)e(subset)h (conforming)75 1317 y(implemen)o(tation.)28 b(It)18 b(is)g(hop)q(ed)g (that)f(an)g(o\016cially)i(sanctioned)g(subset)e(will)i(encourage)f (and)g(allo)o(w)75 1373 y(implemen)o(tors)j(to)e(in)o(tro)q(duce)i(MPI) f(in)g(a)g(more)g(timely)g(fashion.)35 b(It)20 b(should)h(b)q(e)f (noted,)h(ho)o(w)o(ev)o(er,)75 1429 y(that)d(implemen)o(tation)h(of)f (the)g(subset)h(do)q(es)g(not)e(mak)o(e)h(an)g(implemen)o(tation)i (conform)e(with)g(MPI.)75 1486 y(The)h(subset)g(is)g(a)f(p)q(oten)o (tial)h(\014rst)g(step)f(in)i(the)f(pro)q(cess.)30 b(All)20 b(implemen)o(tations)g(shall)g(conform)e(to)75 1542 y(the)f(en)o(tire)g (standard;)f(implemen)o(tations)i(are)e(encouraged)h(to)f(do)g(the)h (full)g(standard)f(as)h(rapidly)g(as)75 1599 y(p)q(ossible.)166 1658 y(The)f(subset)g(presen)o(ted)g(is)h(consisten)o(t)f(with)g(the)g (complete)h(MPI)f(standard.)21 b(This)16 b(w)o(as)g(an)f(im-)75 1714 y(p)q(ortan)o(t)e(goal)h(so)g(the)h(additional)g(MPI)f(features)g (could)i(b)q(e)e(added)h(without)g(c)o(hanging)f(an)o(y)g(previous)75 1771 y(functionalit)o(y)g(from)e(the)h(user's)g(p)q(ersp)q(ectiv)o(e.) 20 b(Th)o(us,)13 b(users)g(can)g(b)q(e)g(assured)g(that)f(programs)g (written)75 1827 y(no)o(w)f(for)h(the)g(subset)g(will)h(run)f(without)g (mo)q(di\014cation)h(under)g(the)f(full)h(MPI)f(standard.)18 b(F)l(urthermore,)75 1884 y(using)i(additional)g(features)f(of)f(the)h (full)i(MPI)e(standard)f(in)i(the)f(future)g(will)i(not)d(require)i(c)o (hanges)75 1940 y(to)e(co)q(de)i(already)f(written.)30 b(This)20 b(requires)f(that)g(the)f(MPI)h(functions)h(ha)o(v)o(e)e (their)i(full)g(calling)g(ar-)75 1997 y(gumen)o(ts)e(in)h(the)f (subset.)28 b(T)l(o)18 b(ac)o(hiev)o(e)h(this)f(goal,)g(the)g(subset)h (will)g(generally)h(limit)f(access)f(to)g(full)75 2053 y(MPI)d(functionalit)o(y)h(through)f(complete)g(elimination)i(of)e (functions)g(or)g(limitations)h(on)f(the)g(creation)75 2110 y(mec)o(hanisms)h(for)e(certain)i(MPI)f(input)h(argumen)o(ts.)75 2267 y Ft(9.2)59 b(Criteria)21 b(and)e(Rationale)75 2374 y FC(Ha)o(ving)g(the)g(subset)g(b)q(e)h(consisten)o(t)f(with)g(the)g (en)o(tire)g(MPI)g(standard)g(w)o(as)f(considered)i(critical)h(to)75 2430 y(the)16 b(e\013ort.)22 b(In)16 b(addition)i(to)d(this,)h(there)h (w)o(ere)e(man)o(y)h(p)q(ossible)i(criteria)f(to)e(use)h(in)h (determining)h(the)75 2487 y(elemen)o(ts)d(of)e(MPI)h(to)f(include)j (in)f(the)f(subset.)20 b(The)14 b(main)g(criteria)h(used)f(w)o(ere)g (that)f(the)h(MPI)g(subset)75 2543 y(should:)131 2647 y(1.)22 b(Con)o(tain)12 b(routines)h(that)e(are)h(as)g(close)h(as)f(p)q (ossible)i(to)e(curren)o(t)g(standard)g(practice)h(to)f(minimize)189 2704 y(the)j(e\013ort)f(to)h(p)q(ort)f(co)q(des.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 167 175 167 174 bop 75 -100 a Fs(9.3.)34 b(SUBSET)16 b(FUNCTIONALITY)1036 b FC(167)131 45 y(2.)22 b(Con)o(tain)15 b(new)g(and)g(imp)q(ortan)o(t)g (MPI)g(features.)131 164 y(3.)22 b(Allo)o(w)17 b(dev)o(elop)q(ers)h(to) e(b)q(e)i(able)f(to)g(ha)o(v)o(e)f(the)h(subset)g(sho)o(w)f(up)i(as)e (rapidly)i(as)f(p)q(ossible)h(while)189 220 y(still)e(meeting)g(the)f (other)g(criteria.)166 338 y(There)i(w)o(ere)g(man)o(y)f(rationales)h (for)g(these)g(criteria.)26 b(It)17 b(w)o(as)f(recognized)i(that)e(it)h (is)h(imp)q(ossible)75 395 y(to)d(come)g(up)h(with)f(an)g(ideal)i (subset,)e(and)g(compromises)h(w)o(ere)f(necessary)l(.)20 b(It)c(w)o(as)e(felt)i(that)e(curren)o(t)75 451 y(users)22 b(should)g(b)q(e)h(comfortable)e(in)h(migrating)g(to)f(MPI.)g(Th)o(us,) i(the)f(subset)f(should)i(con)o(tain)f(the)75 508 y(message)16 b(passing)h(elemen)o(ts)g(that)f(represen)o(t)g(common)g(practice)h(to) q(da)o(y)l(.)23 b(F)l(urthermore,)16 b(it)h(w)o(as)f(felt)75 564 y(that)22 b(the)h(initial)h(\(and)f(later\))f(v)o(ersion)h(of)f (MPI)h(should)g(main)o(tain)g(e\016ciency;)28 b(otherwise,)c(users)75 621 y(could)18 b(su\013er)f(from)g(disenc)o(han)o(tmen)o(t)h(with)g (MPI)f(on)g(their)h(\014rst)f(usage)g(and)h(migh)o(t)f(nev)o(er)g(giv)o (e)h(the)75 677 y(standard)12 b(a)g(second)g(try)l(.)19 b(Ha)o(ving)12 b(the)h(common)e(practice)i(routines)g(in)g(the)f (subset)h(should)g(encourage)75 734 y(dev)o(elop)q(ers)i(to)e(optimize) i(these)f(routines)g(in)g(the)g(initial)i(v)o(ersion)e(of)f(MPI.)g (Also,)h(although)g(some)g(felt)75 790 y(that)g(the)g(p)q(ortabilit)o (y)h(o\013ered)f(b)o(y)g(the)h(MPI)f(subset)h(w)o(as)e(su\016cien)o(t)i (to)f(en)o(tice)h(users,)f(man)o(y)g(felt)h(that)75 846 y(the)j(subset)g(should)g(con)o(tain)g(some)g(new)g(features)f(o)o(v)o (er)g(and)h(ab)q(o)o(v)o(e)f(curren)o(t)h(common)f(practice)h(as)75 903 y(an)i(added)h(incen)o(tiv)o(e.)36 b(The)20 b(additional)h(items)g (selected)g(w)o(ere)f(mean)o(t)f(to)g(represen)o(t)i(some)e(of)h(the)75 959 y(signi\014can)o(t)e(new)f(features)f(of)h(MPI.)f(Balanced)i (against)f(these)g(\014rst)f(t)o(w)o(o)g(goals)h(is)g(the)g(need)h(for) e(the)75 1016 y(subset)e(to)f(sho)o(w)g(up)i(in)f(a)g(timely)h (fashion.)k(Th)o(us,)14 b(eac)o(h)g(feature)g(c)o(hosen)g(for)f (inclusion)j(in)f(the)f(subset)75 1072 y(w)o(as)21 b(deemed)h(of)f (su\016cien)o(t)h(imp)q(ortance)g(to)f(out)o(w)o(eigh)h(its)f(added)h (complexit)o(y)h(in)f(implemen)o(ting)75 1129 y(the)c(subset.)27 b(Although)18 b(some)f(functions)i(seem)f(easy)f(to)g(implemen)o(t,)i (there)f(are)f(often)g(o)o(v)o(erlo)q(ok)o(ed)75 1185 y(costs)j(in)h(testing,)g(do)q(cumen)o(tation,)h(and)f(dev)o(elopmen)o (t)g(that)f(w)o(ere)g(considered)h(b)q(efore)g(a)f(feature)75 1242 y(w)o(as)15 b(added)i(to)f(the)g(MPI)h(subset.)23 b(Inclusion)c(of)d(to)q(o)f(man)o(y)h(routines)h(migh)o(t)f(lead)h(to)f (a)g(mo)q(derately)75 1298 y(e\016cien)o(t)21 b(implemen)o(tation)h(of) e(them)h(all)g(instead)h(of)e(a)g(v)o(ery)g(e\016cien)o(t)i(and)f(p)q (ossibly)h(more)e(useful)75 1355 y(implemen)o(tation)c(of)f(a)g (smaller)h(subset.)75 1533 y Ft(9.3)59 b(Subset)19 b(F)n(unctionalit)n (y)75 1646 y FC(The)c(follo)o(wing)g(sections)g(con)o(tain)g(a)f (general)i(summary)e(and)g(sp)q(eci\014c)j(routines)e(of)f(what)g(is)h (included)75 1703 y(in/excluded)f(from)e(the)g(subset)g(for)f(eac)o(h)h (c)o(hapter.)19 b(The)12 b(general)g(requiremen)o(ts)h(of)e(eac)o(h)h (c)o(hapter)g(also)75 1759 y(hold)h(for)f(the)h(subset.)19 b(As)12 b(examples,)h(the)g(message)f(en)o(v)o(elop)q(e)h(in)h (subsection)f(3.2.2,)e(the)i(seman)o(tics)f(in)75 1816 y(section)j(3.4,)f(the)h(t)o(yp)q(e)g(matc)o(hing)g(in)g(section)h (3.5,)d(and)j(the)e(comm)o(unication)i(ob)s(jects)e(in)i(subsection)75 1872 y(3.8.1)e(are)g(all)j(part)d(of)h(the)g(subset.)75 2029 y Fp(9.3.1)49 b(P)o(oint)16 b(to)h(P)o(oint)f(F)o(unctionalit)o(y) 75 2127 y FC(The)j(p)q(oin)o(t)h(to)e(p)q(oin)o(t)i(routines)f(are)g (not)g(only)h(cen)o(tral)f(to)f(MPI)h(but)h(represen)o(t)f(common)f (practice)75 2183 y(to)q(da)o(y)l(.)k(As)16 b(suc)o(h,)g(the)h(ma)s (jorit)o(y)d(of)i(the)g(functionalit)o(y)h(in)g(this)g(c)o(hapter)f(is) g(included)j(in)e(the)f(subset.)75 2240 y(Deriv)o(ed)f(datat)o(yp)q(es) e(represen)o(t)h(a)g(signi\014can)o(t)h(new)f(feature)g(in)h(MPI)f(and) h(is)f(included)j(in)e(the)f(subset.)75 2296 y(Ho)o(w)o(ev)o(er,)20 b(only)h(basic)g(datat)o(yp)q(es)f(of)g(section)h(3.2.1)e(will)j(b)q(e) f(allo)o(w)o(ed)g(as)f(input)h(to)f(the)g(datat)o(yp)q(e)75 2353 y(construction)15 b(routines)h(describ)q(ed)h(in)f(section)g (3.13.1.)166 2415 y(T)l(o)e(reduce)h(the)g(di\016cult)o(y)g(of)f (implemen)o(tation,)i(sev)o(eral)e(features)g(w)o(ere)g(left)h(out)f (of)g(the)g(subset.)75 2472 y(The)19 b(abilit)o(y)h(to)e(test)h(for)f (m)o(ultiple)j(completions)e(of)g(messages)f(is)i(remo)o(v)o(ed.)30 b(Also,)20 b(the)f(abilit)o(y)h(to)75 2528 y(cancel)13 b(a)e(message)g(and)g(use)h(p)q(ersisten)o(t)g(comm)o(unication)g(ob)s (jects)f(is)h(remo)o(v)o(ed.)18 b(Finally)l(,)c(the)e(routines)75 2585 y(asso)q(ciated)j(with)h(ready)f(and)g(sync)o(hronous)h(comm)o (unication)g(mo)q(des)f(are)g(excluded.)166 2647 y(The)f(table)h(b)q (elo)o(w)g(con)o(tains)f(a)g(listing)i(b)o(y)e(section)h(of)f(whic)o(h) h(routines)f(from)g(the)g(p)q(oin)o(t)h(to)e(p)q(oin)o(t)75 2704 y(comm)o(unication)j(c)o(hapter)f(are)g(included)j(in/excluded)f (from)e(the)g(subset.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 168 176 168 175 bop 75 -100 a FC(168)593 b Fs(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)120 45 y FC(Section)62 b(Title)400 b(Included)414 b(Excluded)p 75 68 1886 2 v 120 120 a(3.2)146 b(Basic)16 b(Send)g(Op)q(eration)63 b Fv(MPI)p 905 120 14 2 v 16 w(SEND)367 b FC(||)120 189 y(3.2.1)110 b(Message)15 b(Data)217 b(All)17 b(t)o(yp)q(es)402 b(||)120 258 y(3.3)146 b(Basic)16 b(Recv)g(Op)q(eration)61 b Fv(MPI)p 905 258 V 16 w(RECV)369 b FC(||)120 327 y(3.3.1)110 b(Return)16 b(Status)213 b Fv(MPI)p 905 327 V 16 w(GET)p 1009 327 V 17 w(SOURCE)205 b FC(||)820 383 y Fv(MPI)p 905 383 V 16 w(GET)p 1009 383 V 17 w(T)l(A)o(G)820 439 y(MPI)p 905 439 V 16 w(GET)p 1009 439 V 17 w(COUNT)120 508 y FC(3.7)146 b(Comm.)19 b(Mo)q(des)202 b(Standard)400 b Fv(MPI)p 1487 508 V 16 w(RSEND)1402 565 y(MPI)p 1487 565 V 16 w(SSEND)120 634 y FC(3.8.2)110 b(Comm.)19 b(Initiation)147 b Fv(MPI)p 905 634 V 16 w(ISEND)354 b(MPI)p 1487 634 V 16 w(IRSEND)820 690 y(MPI)p 905 690 V 16 w(IRECV)i(MPI)p 1487 690 V 16 w(ISSEND)120 759 y FC(3.8.3)110 b(Comm.)19 b(Completion)102 b Fv(MPI)p 905 759 V 16 w(W)l(AIT)371 b FC(||)820 816 y Fv(MPI)p 905 816 V 16 w(TEST)120 884 y FC(3.8.5)110 b(Multiple)17 b(Completion)83 b(||)492 b Fv(MPI)p 1487 884 V 16 w(W)l(AIT)l(ANY)1402 941 y(MPI)p 1487 941 V 16 w(TEST)l(ANY)1402 997 y(MPI)p 1487 997 V 16 w(W)l(AIT)l(ALL)1402 1054 y(MPI)p 1487 1054 V 16 w(TEST)l(ALL)120 1123 y FC(3.9)146 b(Prob)q(e)15 b(&)h(Cancel)178 b Fv(MPI)p 905 1123 V 16 w(PROBE)336 b(MPI)p 1487 1123 V 16 w(CANCEL)820 1179 y(MPI)p 905 1179 V 16 w(IPROBE)323 b(MPI)p 1487 1179 V 16 w(TEST)p 1617 1179 V 17 w(CANCELLED)120 1248 y FC(3.10)123 b(P)o(ersisten)o(t)15 b(Comm.)139 b(||)492 b Fv(MPI)p 1487 1248 V 16 w(CREA)l(TE)p 1672 1248 V 18 w(SEND)355 1305 y FC(Ob)s(jects)894 b Fv(MPI)p 1487 1305 V 16 w(CREA)l(TE)p 1672 1305 V 18 w(RSEND)1402 1361 y(MPI)p 1487 1361 V 16 w(CREA)l(TE)p 1672 1361 V 18 w(SSEND)1402 1417 y(MPI)p 1487 1417 V 16 w(CREA)l(TE)p 1672 1417 V 18 w(RECV)1402 1474 y(MPI)p 1487 1474 V 16 w(ST)l(ART)1402 1530 y(MPI)p 1487 1530 V 16 w(ST)l(ART)l(ALL)1402 1587 y(MPI)p 1487 1587 V 16 w(REQUEST)p 1706 1587 V 18 w(FREE)120 1656 y FC(3.11)123 b(Send-receiv)o(e)252 b Fv(MPI)p 905 1656 V 16 w(SENDRECV)g(MPI)p 1487 1656 V 16 w(SENDRECV)p 1735 1656 V 18 w(REPLA)o(CE)120 1725 y FC(3.12)123 b(Null)17 b(pro)q(cess)252 b Fl(MPI)p 897 1725 13 2 v 15 w(PROCNULL)278 b FC(||)120 1794 y(3.13.1)87 b(Datet)o(yp)q(e)15 b(const.)180 b Fv(MPI)p 905 1794 14 2 v 16 w(TYPE)p 1038 1794 V 17 w(CONTIGUOUS)62 b FC(See)16 b(text)f(ab)q(o)o(v)o(e)820 1850 y Fv(MPI)p 905 1850 V 16 w(TYPE)p 1038 1850 V 17 w(VECTOR)820 1906 y(MPI)p 905 1906 V 16 w(TYPE)p 1038 1906 V 17 w(HVECTOR)820 1963 y(MPI)p 905 1963 V 16 w(TYPE)p 1038 1963 V 17 w(INDEXED)820 2019 y(MPI)p 905 2019 V 16 w(TYPE)p 1038 2019 V 17 w(HINDEXED)820 2076 y(MPI)p 905 2076 V 16 w(TYPE)p 1038 2076 V 17 w(STRUCT)120 2145 y FC(3.13.2)87 b(Address)16 b(and)f(exten)o(t)109 b Fv(MPI)p 905 2145 V 16 w(ADDRESS)355 2201 y FC(functions)285 b Fv(MPI)p 905 2201 V 16 w(TYPE)p 1038 2201 V 17 w(EXTENT)820 2258 y(MPI)p 905 2258 V 16 w(TYPE)p 1038 2258 V 17 w(SIZE)820 2314 y(MPI)p 905 2314 V 16 w(TYPE)p 1038 2314 V 17 w(COUNT)120 2383 y FC(3.13.3)87 b(Lo)o(w)o(er-b)q(ound)16 b(and)147 b Fv(MPI)p 905 2383 V 16 w(LB)355 2439 y FC(upp)q(er-b)q(ound)212 b Fv(MPI)p 905 2439 V 16 w(UB)120 2508 y FC(3.13.4)87 b(Commit)15 b(and)g(free)158 b Fv(MPI)p 905 2508 V 16 w(TYPE)p 1038 2508 V 17 w(COMMIT)164 b FC(||)820 2565 y Fv(MPI)p 905 2565 V 16 w(TYPE)p 1038 2565 V 17 w(FREE)120 2634 y FC(3.13.5)87 b(Use)15 b(of)g(General...)163 b Fv(MPI)p 905 2634 V 16 w(GET)p 1009 2634 V 17 w(ELEMENTS)145 b FC(||)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 169 177 169 176 bop 75 -100 a Fs(9.3.)34 b(SUBSET)16 b(FUNCTIONALITY)1036 b FC(169)75 45 y Fp(9.3.2)49 b(Collective)17 b(Communication)g(F)o (unctionalit)o(y)75 143 y FC(As)h(with)f(p)q(oin)o(t)h(to)f(p)q(oin)o (t)h(functionalit)o(y)l(,)i(collectiv)o(e)f(comm)o(unication)f(is)g (common)g(practice)g(to)q(da)o(y)l(,)75 200 y(and)d(the)f(functionalit) o(y)i(is)e(included)j(in)f(the)e(subset.)20 b(F)l(urthermore,)13 b(the)i(abilit)o(y)g(to)f(de\014ne)i(arbitrary)75 256 y(groups)e(in)i(the)f(MPI)g(subset,)f(whic)o(h)i(are)e(then)h(asso)q (ciated)g(with)g(a)g(comm)o(unicator,)f(allo)o(ws)h(the)g(user)75 313 y(to)k(p)q(erform)g(collectiv)o(e)j(comm)o(unication)e(op)q (erations)g(o)o(v)o(er)f(an)g(arbitrary)g(set)h(of)f(pro)q(cesses.)33 b(This)75 369 y(increases)15 b(the)f(functionalit)o(y)h(of)f(the)g (collectiv)o(e)i(comm)o(unication)e(routines)h(o)o(v)o(er)e(what)g(is)i (commonly)75 425 y(a)o(v)m(ailable)i(to)q(da)o(y)l(.)166 488 y(T)l(o)i(ease)h(initial)i(implemen)o(tation,)f(the)f(user)g (de\014ned)h(reduce)f(functions)h(are)e(excluded)j(from)75 545 y(the)d(subset.)33 b(Note)19 b(that)f(all)j(of)d(the)i(system)f (pro)o(vided)h(reduce)g(functions)g(listed)h(in)f(section)g(4.5.1)75 601 y(are)i(include)i(in)f(the)f(subset.)40 b(Also,)24 b(the)e(scan)g(functions)h(are)f(excluded)i(from)d(the)h(subset.)41 b(Fi-)75 658 y(nally)l(,)25 b(the)e(generalized)h(\\v")e(v)o(ersions)h (of)f(routines)h(are)f(excluded)j(from)c(the)i(subset)g(as)f(w)o(ell)h (as)75 714 y Fv(MPI)p 160 714 14 2 v 16 w(REDUCE)p 352 714 V 17 w(SCA)l(TTER)p FC(.)166 777 y(The)f(table)g(b)q(elo)o(w)g(con) o(tains)f(a)g(listing)i(b)o(y)f(section)g(of)f(whic)o(h)h(routines)g (from)f(the)g(collectiv)o(e)75 833 y(comm)o(unication)16 b(c)o(hapter)f(are)g(included)j(in/excluded)f(from)e(the)g(subset.)120 950 y(Section)62 b(Title)459 b(Included)246 b(Excluded)p 75 972 1869 2 v 120 1025 a(4.3)146 b(Barrier)15 b(Sync)o(h.)267 b Fv(MPI)p 964 1025 14 2 v 15 w(BARRIER)130 b FC(||)120 1093 y(4.4.1)110 b(Broadcast)354 b Fv(MPI)p 964 1093 V 15 w(BCAST)172 b FC(||)120 1162 y(4.4.2)110 b(Gather)414 b Fv(MPI)p 964 1162 V 15 w(GA)l(THER)142 b(MPI)p 1378 1162 V 16 w(GA)l(THERV)120 1231 y FC(4.4.4)110 b(Scatter)412 b Fv(MPI)p 964 1231 V 15 w(SCA)l(TTER)119 b(MPI)p 1378 1231 V 16 w(SCA)l(TTERV)120 1300 y FC(4.4.6)110 b(Gather-to-all)296 b Fv(MPI)p 964 1300 V 15 w(ALLGA)l(THER)62 b(MPI)p 1378 1300 V 16 w(ALLGA)l(THERV)120 1369 y FC(4.4.7)110 b(All-to-All)18 b(Scatter/Gather)44 b Fv(MPI)p 964 1369 V 15 w(ALL)l(TO)o(ALL)98 b(MPI)p 1378 1369 V 16 w(ALL)l(TO)o(ALL)-5 b(V)120 1438 y FC(4.5.1)110 b(Reduce)411 b Fv(MPI)p 964 1438 V 15 w(REDUCE)120 1507 y FC(4.5.2)110 b(User-Reduce)720 b Fv(MPI)p 1378 1507 V 16 w(USER)p 1506 1507 V 17 w(REDUCE)1293 1563 y(MPI)p 1378 1563 V 16 w(USER)p 1506 1563 V 17 w(REDUCEA)120 1632 y FC(4.5.3)110 b(All-Reduce)338 b Fv(MPI)p 964 1632 V 15 w(ALLREDUCE)61 b(MPI)p 1378 1632 V 16 w(USER)p 1506 1632 V 17 w(ALLREDUCE)1293 1689 y(MPI)p 1378 1689 V 16 w(USER)p 1506 1689 V 17 w(ALLREDUCEA)120 1758 y FC(4.5.4)110 b(Reduce-Scatter)668 b Fv(MPI)p 1378 1758 V 16 w(REDUCE)p 1570 1758 V 17 w(SCA)l(TTER)1293 1814 y(MPI)p 1378 1814 V 16 w(USER)p 1506 1814 V 17 w(REDUCE)p 1699 1814 V 18 w(SCA)l(TTER)1293 1871 y(MPI)p 1378 1871 V 16 w(USER)p 1506 1871 V 17 w(REDUCE)p 1699 1871 V 18 w(SCA)l(TTERA)120 1939 y FC(4.5.5)110 b(Scan)461 b(||)324 b Fv(MPI)p 1378 1939 V 16 w(SCAN)1293 1996 y(MPI)p 1378 1996 V 16 w(USER)p 1506 1996 V 17 w(SCAN)1293 2052 y(MPI)p 1378 2052 V 16 w(USER)p 1506 2052 V 17 w(SCANA)75 2210 y Fp(9.3.3)49 b(Groups,)15 b(Contexts,)g(Communicato)o(rs,)g(and)i(Cacheing)75 2308 y FC(Groups,)22 b(con)o(texts,)f(and)g(comm)o(unicators)f(are)h (used)g(throughout)g(the)g(MPI)f(standard.)37 b(Though)75 2365 y(these)19 b(concepts)g(are)g(used)h(in)f(some)g(pac)o(k)m(ages)g (to)q(da)o(y)l(,)g(their)g(inclusion)j(in)d(MPI)g(represen)o(ts)g(a)g (sig-)75 2421 y(ni\014can)o(t)e(new)g(feature)g(o)o(v)o(er)f(what)g(is) h(generally)h(a)o(v)m(ailable)g(in)g(common)e(practice)i(to)q(da)o(y)l (.)24 b(F)l(or)16 b(these)75 2478 y(reasons,)i(this)h(functionalit)o(y) g(is)g(included)i(in)e(the)f(subset.)29 b(Ho)o(w)o(ev)o(er,)17 b(implemen)o(tation)j(of)e(groups,)75 2534 y(con)o(texts,)d(and)i(comm) o(unicators)f(is)g(also)g(a)g(signi\014can)o(t)i(e\013ort.)j(As)16 b(a)g(compromise,)h(the)f(MPI)g(in)o(tra-)75 2591 y(comm)o(unicator)21 b(functionalit)o(y)i(is)f(included,)k(but)21 b(the)h(in)o(ter-comm)o (unicator)g(functionalit)o(y)h(is)f(ex-)75 2647 y(cluded.)30 b(The)18 b(cac)o(heing)h(facilit)o(y)g(w)o(as)e(included)k(to)c(supp)q (ort)h(the)h(top)q(ology)e(functionalit)o(y)i(and)g(ex-)75 2704 y(tensions)d(to)e(MPI.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 170 178 170 177 bop 75 -100 a FC(170)593 b Fs(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)166 45 y FC(The)f(table)g(b)q(elo)o(w)h(con)o(tains)f(a)f(listing)i(b)o(y)f (section)h(of)e(whic)o(h)i(routines)f(from)f(the)h(Groups,)f(Con-)75 102 y(texts,)g(Comm)o(unicators,)g(and)i(Cac)o(heing)f(c)o(hapter)g (are)g(included)j(in/excluded)g(from)c(the)i(subset.)75 218 y(Section)g(Title)302 b(Included)522 b(Excluded)p 75 240 1924 2 v 75 293 a(5.2.4)64 b(Prede\014ned)17 b(Comm.)26 b Fv(MPI)p 716 293 14 2 v 16 w(COMM)p 874 293 V 16 w(W)o(ORLD)272 b FC(||)75 362 y(5.3.1)64 b(Group)15 b(Accessors)66 b Fv(MPI)p 716 362 V 16 w(GROUP)p 884 362 V 18 w(SIZE)329 b FC(||)631 418 y Fv(MPI)p 716 418 V 16 w(GROUP)p 884 418 V 18 w(RANK)631 475 y(MPI)p 716 475 V 16 w(GROUP)p 884 475 V 18 w(TRANSLA)l(TE)p 1158 475 V 17 w(RANKS)75 543 y FC(5.3.2)64 b(Group)15 b(Const.)125 b Fv(MPI)p 716 543 V 16 w(GROUP)p 884 543 V 18 w(UNION)281 b FC(||)631 600 y Fv(MPI)p 716 600 V 16 w(GROUP)p 884 600 V 18 w(INTERSECTION)631 656 y(MPI)p 716 656 V 16 w(GROUP)p 884 656 V 18 w(DIFFERENCE)631 713 y(MPI)p 716 713 V 16 w(GROUP)p 884 713 V 18 w(INCL)631 769 y(MPI)p 716 769 V 16 w(GROUP)p 884 769 V 18 w(EX)o(CL)631 826 y(MPI)p 716 826 V 16 w(GROUP)p 884 826 V 18 w(RANGE)p 1050 826 V 17 w(INCL)631 882 y(MPI)p 716 882 V 16 w(GROUP)p 884 882 V 18 w(RANGE)p 1050 882 V 17 w(EX)o(CL)75 951 y FC(5.3.3)64 b(Group)15 b(Destruct.)70 b Fv(MPI)p 716 951 V 16 w(GROUP)p 884 951 V 18 w(FREE)313 b FC(||)75 1020 y(5.4.1)64 b(Comm.)19 b(Access.)90 b Fv(MPI)p 716 1020 V 16 w(COMM)p 874 1020 V 16 w(SIZE)341 b FC(||)631 1076 y Fv(MPI)p 716 1076 V 16 w(COMM)p 874 1076 V 16 w(RANK)631 1133 y(MPI)p 716 1133 V 16 w(COMM)p 874 1133 V 16 w(GROUP)75 1202 y FC(5.4.2)64 b(Comm.)19 b(Const.)103 b Fv(MPI)p 716 1202 V 16 w(COMM)p 874 1202 V 16 w(DUP)341 b FC(||)631 1258 y Fv(MPI)p 716 1258 V 16 w(COMM)p 874 1258 V 16 w(MAKE)631 1315 y(MPI)p 716 1315 V 16 w(COMM)p 874 1315 V 16 w(SPLIT)75 1384 y FC(5.4.3)64 b(Comm.)19 b(Destruct.)48 b Fv(MPI)p 716 1384 V 16 w(COMM)p 874 1384 V 16 w(FREE)325 b FC(||)75 1453 y(5.5.1)64 b(Comm.)19 b(Access.)90 b(||)600 b Fv(MPI)p 1406 1453 V 16 w(COMM)p 1564 1453 V 16 w(TEST)p 1694 1453 V 17 w(INTER)75 1521 y FC(5.5.2)64 b(In)o(tercomm.)20 b(Const.)f(||)600 b Fv(MPI)p 1406 1521 V 16 w(INTERCOMM)p 1696 1521 V 16 w(ST)l(ART)264 1578 y FC(&)16 b(Destr.)884 b Fv(MPI)p 1406 1578 V 16 w(INTERCOMM)p 1696 1578 V 16 w(FINISH)1321 1634 y(MPI)p 1406 1634 V 16 w(INTER)p 1554 1634 V 16 w(COMM)p 1712 1634 V 17 w(MERGE)75 1703 y FC(5.5.3)64 b(Name)15 b(Service)128 b(||)600 b Fv(MPI)p 1406 1703 V 16 w(INTERCOMM)p 1696 1703 V 16 w(NAME)p 1841 1703 V 17 w(ST)l(ART)75 1772 y FC(5.6.1)64 b(F)l(unctionalit)o(y)135 b Fv(MPI)p 716 1772 V 16 w(A)l(TTR)p 849 1772 V 17 w(GET)p 954 1772 V 17 w(KEY)264 b FC(||)631 1829 y Fv(MPI)p 716 1829 V 16 w(A)l(TTR)p 849 1829 V 17 w(FREE)p 975 1829 V 17 w(KEY)631 1885 y(MPI)p 716 1885 V 16 w(A)l(TTR)p 849 1885 V 17 w(PUT)p 957 1885 V 16 w(V)l(ALUE)631 1942 y(MPI)p 716 1942 V 16 w(A)l(TTR)p 849 1942 V 17 w(GET)p 954 1942 V 17 w(V)l(ALUE)631 1998 y(MPI)p 716 1998 V 16 w(A)l(TTR)p 849 1998 V 17 w(DELETE)p 1036 1998 V 16 w(V)l(ALUE)75 2276 y Fp(9.3.4)49 b(T)l(op)q(ology)19 b(F)o(unctionalit)o(y)75 2401 y FC(The)d(capabilit)o(y)h(presen)o(ted)f (in)g(the)g(pro)q(cess)f(top)q(ology)h(c)o(hapter)f(represen)o(ts)g(a)h (new)f(functionalit)o(y)i(in)75 2458 y(MPI)i(and)g(is)h(not)f (generally)h(common)f(practice)h(to)q(da)o(y)l(.)31 b(T)l(o)19 b(mak)o(e)f(this)i(a)o(v)m(ailable)h(in)f(the)f(subset,)75 2514 y(the)g(functionalit)o(y)i(asso)q(ciated)e(with)g(Cartesian)g(top) q(ologies)h(is)g(included,)i(but)d(the)h(general)f(graph)75 2571 y(top)q(ologies)d(are)f(excluded.)166 2647 y(The)22 b(table)h(b)q(elo)o(w)g(con)o(tains)f(a)g(listing)i(b)o(y)e(section)h (of)f(whic)o(h)h(routines)f(from)g(the)g(top)q(ology)75 2704 y(c)o(hapter)15 b(are)g(included)j(in/excluded)f(from)e(the)g (subset.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 171 179 171 178 bop 75 -100 a Fs(9.3.)34 b(SUBSET)16 b(FUNCTIONALITY)1036 b FC(171)120 45 y(Section)62 b(Title)403 b(Included)285 b(Excluded)p 75 68 1868 2 v 120 120 a(6.5)146 b(T)l(op)q(ology)15 b(Const.)170 b Fv(MPI)p 908 120 14 2 v 16 w(MAKE)p 1053 120 V 16 w(CART)91 b(MPI)p 1361 120 V 16 w(MAKE)p 1506 120 V 16 w(GRAPH)823 176 y(MPI)p 908 176 V 16 w(MAKE)p 1053 176 V 16 w(DIMS)120 245 y FC(6.5.1)110 b(T)l(op)q(ol.)20 b(inquiry)d(func.)98 b Fv(MPI)p 908 245 V 16 w(TOPO)p 1050 245 V 18 w(ST)l(A)l(TUS)823 302 y(MPI)p 908 302 V 16 w(GET)p 1012 302 V 17 w(CARTDIM)45 b(MPI)p 1361 302 V 16 w(GET)p 1465 302 V 17 w(GRAPHDIMS)823 358 y(MPI)p 908 358 V 16 w(GET)p 1012 358 V 17 w(CART)131 b(MPI)p 1361 358 V 16 w(GET)p 1465 358 V 17 w(GRAPH)823 415 y(MPI)p 908 415 V 16 w(CART)p 1043 415 V 17 w(RANK)96 b(MPI)p 1361 415 V 16 w(GRAPH)p 1527 415 V 18 w(NEIGHBOR)p 1771 415 V 18 w(COUNT)823 471 y(MPI)p 908 471 V 16 w(CART)p 1043 471 V 17 w(COORD)62 b(MPI)p 1361 471 V 16 w(GRAPH)p 1527 471 V 18 w(NEIGHBORS)120 540 y FC(6.5.2)110 b(Cart.)19 b(Shift)d(Co)q(ord.)128 b Fv(MPI)p 908 540 V 16 w(CART)p 1043 540 V 17 w(SHIFT)120 609 y FC(6.5.3)110 b(P)o(art.)19 b(of)14 b(Cart.)19 b(Struct.)65 b Fv(MPI)p 908 609 V 16 w(CART)p 1043 609 V 17 w(SUB)120 678 y FC(6.5.4)110 b(Lo)o(w{lev)o(el)16 b(top)q(ol.)k(func.)57 b Fv(MPI)p 908 678 V 16 w(MAP)p 1023 678 V 17 w(CART)120 b(MPI)p 1361 678 V 16 w(MAP)p 1476 678 V 17 w(GRAPH)75 796 y Fp(9.3.5)49 b(Language)18 b(Binding)75 882 y FC(The)e(subset)f(shall)i (con)o(tain)e(the)g(F)l(ortran)g(77)f(and)i(C)f(language)h(bindings.)22 b(All)16 b(routines)g(included)i(in)75 938 y(the)d(subset)h(will)h (conform)d(to)h(the)g(rules)h(stated)f(in)h(c)o(hapter)f(2)g(and)g (annex)g(A.)75 1056 y Fp(9.3.6)49 b(MPI)17 b(Environmental)g (Management)75 1142 y FC(The)i(abilit)o(y)i(for)d(the)i(user)f(to)g (mak)o(e)f(inquiries)k(of)c(the)i(MPI)f(system)g(is)g(an)g(imp)q(ortan) o(t)g(feature)g(in)75 1198 y(p)q(ortable)f(programming.)26 b(F)l(or)16 b(this)i(reason,)f(most)g(of)g(the)g(functions)h(are)f (included)j(in)f(the)e(subset.)75 1255 y(Ho)o(w)o(ev)o(er,)e(the)i (abilit)o(y)g(for)f(the)g(user)h(to)f(sp)q(ecify)h(an)g(error)e (handler)j(is)f(not)f(included)j(in)e(the)f(subset.)75 1311 y(This)g(is)f(consisten)o(t)h(with)f(not)g(including)j(user)e (called)g(functions)g(in)g(the)g(subset.)166 1368 y(The)e(table)h(b)q (elo)o(w)g(con)o(tains)f(a)g(listing)i(b)o(y)e(section)h(of)f(whic)o(h) h(routines)f(from)g(the)g(MPI)h(En)o(viron-)75 1424 y(men)o(tal)g (Managemen)o(t)f(c)o(hapter)h(are)g(included)j(in/excluded)g(from)c (the)i(subset.)75 1541 y(Section)g(Title)222 b(Included)513 b(Excluded)p 75 1563 1785 2 v 75 1615 a(7.1.1)64 b(En)o(vir.)20 b(Inquires)d Fv(MPI)p 636 1615 14 2 v 16 w(GET)p 740 1615 V 16 w(V)l(ALID)p 883 1615 V 17 w(T)l(A)o(G)p 986 1615 V 17 w(RANGE)84 b FC(||)551 1672 y Fv(MPI)p 636 1672 V 16 w(GET)p 740 1672 V 16 w(PROCESSOR)p 1015 1672 V 19 w(NAME)551 1728 y(MPI)p 636 1728 V 16 w(GET)p 740 1728 V 16 w(HOST)75 1797 y FC(7.1.2)64 b(Bu\013ering)135 b Fv(MPI)p 636 1797 V 16 w(GET)p 740 1797 V 16 w(BUFFER)p 925 1797 V 18 w(P)l(ARAMS)113 b(MPI)p 1317 1797 V 16 w(USER)p 1445 1797 V 17 w(SPECIFIES)p 1676 1797 V 16 w(BUFFER)551 1854 y(MPI)p 636 1854 V 16 w(SUGGEST)p 851 1854 V 17 w(BUFFER)p 1037 1854 V 18 w(P)l(ARAMS)75 1923 y FC(7.2)100 b(Error)14 b(handling)25 b Fv(MPI)p 636 1923 V 16 w(GET)p 740 1923 V 16 w(ERRORMODE)551 1979 y(MPI)p 636 1979 V 16 w(SET)p 735 1979 V 16 w(ERRORMODE)204 b(MPI)p 1317 1979 V 16 w(SET)p 1416 1979 V 16 w(ERRHANDLER)551 2036 y(MPI)p 636 2036 V 16 w(ERROR)p 799 2036 V 18 w(STRING)75 2104 y FC(7.3)100 b(Startup)165 b Fv(MPI)p 636 2104 V 16 w(INIT)494 b FC(||)551 2161 y Fv(MPI)p 636 2161 V 16 w(FINALIZE)551 2217 y(MPI)p 636 2217 V 16 w(INITIALIZED)551 2274 y(MPI)p 636 2274 V 16 w(ABORT)75 2392 y Fp(9.3.7)49 b(Pro\014ling)18 b(F)o(unctionalit)o(y)75 2478 y FC(Pro\014ling)c(is)g (a)e(capabilit)o(y)j(that)d(will)j(b)q(e)f(of)e(great)g(utilit)o(y)j (in)f(MPI)f(for)f(determining)j(the)e(p)q(erformance)75 2534 y(of)g(eac)o(h)g(implemen)o(tation.)20 b(F)l(urthermore,)12 b(although)i(implemen)o(tation)g(of)e(the)h(pro\014ling)i(capabilities) 75 2591 y(doubles)k(the)f(n)o(um)o(b)q(er)g(of)f(routines)h(supplied,)j (these)d(routines)g(can)g(easily)h(b)q(e)f(generated)g(from)f(the)75 2647 y(routines)11 b(required)h(elsewhere)g(in)g(the)f(subset.)18 b(Giv)o(en)11 b(these)g(considerations,)i(the)d(subset)h(will)i(con)o (tain)75 2704 y(the)i(shado)o(w)g(routines)h(\\PMPI)p 635 2704 V 16 w(")e(for)h(eac)o(h)g(routine)h(included)i(in)e(the)f (subset.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 172 180 172 179 bop 75 -100 a FC(172)593 b Fs(CHAPTER)16 b(9.)34 b(INITIAL)17 b(IMPLEMENT)l(A)l(TION)f(SUBSET)75 45 y Ft(9.4)59 b(Subset)19 b(T)-5 b(esting)75 147 y FC(The)12 b(test)g(suite)h(is)g(exp)q(ected)g(to)f(include)i(a)e(subset)g(that)g (will)i(v)o(erify)e(the)g(functionalit)o(y)i(of)e(this)g(subset.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 173 181 173 180 bop 75 379 a Fy(Biblio)q(graph)m(y)98 604 y FC([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 661 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 717 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 820 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 876 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 933 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1035 y([3])j(A.)14 b(Beguelin,)j(J.)d(Dongarra,)e(A.)i(Geist,)h(R.)f (Manc)o(hek,)g(and)g(V.)g(Sunderam.)19 b(Visualization)e(and)168 1092 y(debugging)g(in)f(a)f(heterogeneous)g(en)o(vironmen)o(t.)20 b Fr(IEEE)15 b(Computer)p FC(,)h(26\(6\):88{95,)11 b(June)17 b(1993.)98 1194 y([4])k(Luc)14 b(Bomans)f(and)h(Rolf)f(Hemp)q(el.)18 b(The)c(Argonne/GMD)e(macros)g(in)i(F)o(OR)l(TRAN)g(for)e(p)q(ortable) 168 1251 y(parallel)20 b(programming)e(and)h(their)f(implemen)o(tation) i(on)e(the)h(In)o(tel)g(iPSC/2.)29 b Fr(Par)n(al)r(lel)19 b(Com-)168 1307 y(puting)p FC(,)c(15:119{132,)d(1990.)98 1410 y([5])21 b(R.)14 b(Butler)h(and)f(E.)f(Lusk.)18 b(User's)13 b(guide)i(to)e(the)h(P4)g(programming)f(system.)k(T)l(ec)o (hnical)f(Rep)q(ort)168 1466 y(TM-ANL{92/17,)e(Argonne)h(National)h (Lab)q(oratory)l(,)e(1992.)98 1569 y([6])21 b(Rob)q(ert)i(Calkin,)i (Rolf)f(Hemp)q(el,)h(Hans-Christian)f(Hopp)q(e,)g(and)f(P)o(eter)f (Wypior.)42 b(P)o(ortable)168 1626 y(programming)17 b(with)g(the)g (parmacs)g(message{passing)f(library)l(.)27 b Fr(Par)n(al)r(lel)17 b(Computing,)h(sp)n(e)n(cial)168 1682 y(issue)e(on)g(message{p)n (assing)f(interfac)n(es)p FC(,)e(to)i(app)q(ear.)98 1785 y([7])21 b(S.)c(Chittor)g(and)h(R.)f(J.)g(En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d(mesh{connected)i(w)o (ormhole{)168 1841 y(routed)f(net)o(w)o(orks)e(for)i(in)o(terpro)q (cessor)f(comm)o(unication)i(in)f(m)o(ulticomputers.)25 b(In)17 b Fr(Pr)n(o)n(c)n(e)n(e)n(dings)168 1898 y(of)g(the)f(1990)i (Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p FC(,)c(pages)j(647{656,)e (1990.)98 2000 y([8])21 b(S.)f(Chittor)g(and)g(R.)g(J.)f(En)o(b)q(o)q (dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g(of)f(mapping)i(on)e(the)h (comm)o(unica-)168 2057 y(tion)g(p)q(erformance)g(of)f(large)g(m)o (ulticomputers.)34 b(In)20 b Fr(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f (1991)h(International)168 2113 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n(essing,)f(vol.)i(II)f(\(Softwar)n (e\))p FC(,)f(pages)h(I)q(I{1)h({)f(I)q(I{4,)g(1991.)98 2216 y([9])21 b(J.)e(Dongarra,)f(A.)g(Geist,)h(R.)g(Manc)o(hek,)g(and)g (V.)f(Sunderam.)31 b(In)o(tegrated)18 b(PVM)g(framew)o(ork)168 2272 y(supp)q(orts)c(heterogeneous)g(net)o(w)o(ork)f(computing.)18 b Fr(Computers)e(in)e(Physics)p FC(,)f(7\(2\):166{75,)e(April)168 2329 y(1993.)75 2432 y([10])21 b(J.)c(J.)g(Dongarra,)f(R.)h(Hemp)q(el,) i(A.)e(J.)g(G.)f(Hey)l(,)i(and)f(D.)f(W.)h(W)l(alk)o(er.)25 b(A)17 b(prop)q(osal)g(for)g(a)g(user-)168 2488 y(lev)o(el,)c(message)d (passing)h(in)o(terface)g(in)h(a)e(distributed)i(memory)e(en)o (vironmen)o(t.)j(T)l(ec)o(hnical)g(Rep)q(ort)168 2544 y(TM-12231,)g(Oak)j(Ridge)g(National)g(Lab)q(oratory)l(,)e(F)l(ebruary) h(1993.)75 2647 y([11])21 b(Edin)o(burgh)g(P)o(arallel)f(Computing)g (Cen)o(tre,)g(Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fr(CHIMP)19 b(Conc)n(epts)p FC(,)168 2704 y(June)e(1991.)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 174 182 174 181 bop 75 -100 a FC(174)1368 b Fs(BIBLIOGRAPHY)75 45 y FC([12])21 b(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f (Univ)o(ersit)o(y)h(of)g(Edin)o(burgh.)22 b Fr(CHIMP)15 b(V)m(ersion)h(1.0)168 102 y(Interfac)n(e)p FC(,)e(Ma)o(y)g(1992.)75 195 y([13])21 b(D.)d(F)l(eitelson.)28 b(Comm)o(unicators:)d(Ob)s (ject-based)18 b(m)o(ultipart)o(y)g(in)o(teractions)g(for)g(parallel)h (pro-)168 252 y(gramming.)j(T)l(ec)o(hnical)c(Rep)q(ort)f(91-12,)e (Dept.)g(Computer)h(Science,)i(The)e(Hebrew)g(Univ)o(ersit)o(y)168 308 y(of)f(Jerusalem,)h(No)o(v)o(em)o(b)q(er)f(1991.)75 402 y([14])21 b(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o (eyton,)f(and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f(PICL:)g(a)168 459 y(p)q(ortable)f(instrumen)o (ted)f(comm)o(unication)h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e (TM-11616,)f(Oak)h(Ridge)168 515 y(National)k(Lab)q(oratory)l(,)e (Octob)q(er)i(1990.)75 609 y([15])21 b(nCUBE)16 b(Corp)q(oration.)j Fr(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i(r2.0)p FC(,)f(Decem)o(b)q(er)f(1990.)75 703 y([16])21 b(H.)c(M)q(\177)-24 b(uhlen)o(b)q(ein)19 b(O.)d(Kr\177)-23 b(amer.)24 b(Mapping)16 b(strategies)g(in)i(message{based)e(m)o(ultipro)q(cessor)h(sys-)168 759 y(tems.)j Fr(Par)n(al)r(lel)15 b(Computing)p FC(,)g(9:213{225,)d (1989.)75 853 y([17])21 b(P)o(arasoft)14 b(Corp)q(oration.)22 b Fr(Expr)n(ess)16 b(V)m(ersion)g(1.0:)22 b(A)17 b(Communic)n(ation)g (Envir)n(onment)e(for)j(Par-)168 909 y(al)r(lel)e(Computers)p FC(,)f(1988.)75 1003 y([18])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g(system.)22 b(In)17 b Fr(Pr)n(o)n(c)n(e)n(e)n(dings)e(of)i(the)g(Thir)n(d)g(Confer)n(enc)n(e) e(on)168 1060 y(Hyp)n(er)n(cub)n(e)g(Concurr)n(ent)f(Computers)i(and)f (Applic)n(ations)p FC(,)e(pages)h(384{390.)d(A)o(CM)j(Press,)f(1988.)75 1153 y([19])21 b(A.)14 b(Skjellum)i(and)e(A.)g(Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f(comm)o (unication)h(library)168 1210 y(atop)g(the)f(reactiv)o(e)h(k)o(ernel.) 20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g (editors,)g Fr(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 1266 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g (Confer)n(enc)n(e)p FC(,)e(pages)i(767{776.)d(IEEE)168 1323 y(Press,)d(1990.)75 1417 y([20])21 b(A.)12 b(Skjellum,)i(S.)e (Smith,)h(C.)e(Still,)j(A.)d(Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q (co)q(de)g(message)f(passing)168 1473 y(system.)20 b(T)l(ec)o(hnical)d (rep)q(ort,)d(La)o(wrence)i(Liv)o(ermore)f(National)h(Lab)q(oratory)l (,)e(Septem)o(b)q(er)i(1992.)75 1567 y([21])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g(passing)g(in)i(a)e (distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(h-)168 1623 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National)g(Lab)q (oratory)l(,)e(August)h(1992.)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 175 183 175 182 bop 75 358 a Fu(Annex)33 b(A)75 568 y Fy(Language)41 b(Binding)75 810 y Ft(A.1)59 b(Intro)r(duction)75 916 y FC(In)15 b(this)f(section)h(w)o(e)f(summarize)g(the)h(sp)q(eci\014c)h (bindings)g(for)d(b)q(oth)h(F)l(ortran)f(and)h(C.)g(W)l(e)g(presen)o(t) g(\014rst)75 972 y(the)h(C)g(bindings,)i(then)e(the)h(F)l(ortran)e (bindings.)22 b(Listings)16 b(are)f(alphab)q(etical)i(within)f(c)o (hapter.)75 1126 y Ft(A.2)59 b(De\014ned)19 b(Constants)g(fo)n(r)i(C)f (and)f(F)n(o)n(rtran)75 1231 y FC(These)e(are)g(required)h(de\014ned)h (constan)o(ts,)d(to)g(b)q(e)i(de\014ned)h(in)e(the)h(\014les)g Fo(mpi.h)e FC(\(for)g(C\))h(and)g Fo(mpif.h)75 1288 y FC(\(for)d(F)l(ortran\).)75 1405 y Fo(/*)24 b(return)f(codes)g(*/)75 1462 y(MPI_SUCCESS)75 1574 y(/*)h(assorted)e(constants)h(\(both)g(C)h (and)f(Fortran\))g(*/)75 1631 y(MPI_BOTTOM)75 1687 y(MPI_PROCNULL)75 1744 y(MPI_ANY_SOURCE)75 1800 y(MPI_ANY_TAG)75 1857 y(MPI_UNDEFINED)75 1913 y(MPI_UNDEFINED_RANK)75 1970 y(MPI_UB)75 2026 y(MPI_LB)75 2139 y(/*)h(constant)e(array)i(sizes)f(\(Fortran\))f(*/)75 2195 y(MPI_STATUS_SIZE)75 2308 y(/*)i(attributes)e(\(C)i(and)f (Fortran\))g(*/)75 2365 y(MPI_UNDEFINED_ATTR)75 2421 y(MPI_FOUND)75 2478 y(MPI_NOT_FOUND)75 2534 y(MPI_ATTR_COPY_FREELY)75 2591 y(MPI_ATTR_COPY_NEVER)75 2647 y(MPI_ATTRCOPY_CALLBACK)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 176 184 176 183 bop 75 -100 a FC(176)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fo(/*)24 b(Error-handling)e (specifiers)g(\(C)i(and)f(Fortran\))g(*/)75 102 y(MPI_ERRORSFATAL)75 158 y(MPI_ERRORSRETURN)75 214 y(MPI_ERRORSCALLUSERROUTINE)75 327 y(/*)h(Pre-defined)e(error)h(handlers)g(\(C)g(and)h(Fortran\))f(*/) 75 384 y(MPI_NOHANDLER)75 440 y(MPI_DEFAULTHANDLER)75 553 y(/*)h(Maximum)e(sizes)i(for)f(strings)g(*/)75 610 y(MPI_MAX_INTERCOMM_NAME_SIZ)o(E)75 723 y(/*)h(elementary)e(datatypes)h (\(C\))g(*/)75 779 y(MPI_CHAR)75 835 y(MPI_SHORT)75 892 y(MPI_INT)75 948 y(MPI_LONG)75 1005 y(MPI_FLOAT)75 1061 y(MPI_DOUBLE)75 1118 y(MPI_BYTE)75 1174 y(MPI_UNSIGNED_CHAR)75 1231 y(MPI_UNSIGNED_SHORT)75 1287 y(MPI_UNSIGNED_LONG)75 1344 y(MPI_UNSIGNED_INT)75 1456 y(/*)h(elementary)e(datatypes)h (\(Fortran\))f(*/)75 1513 y(MPI_INTEGER)75 1569 y(MPI_REAL)75 1626 y(MPI_DOUBLE_PRECISION)75 1682 y(MPI_COMPLEX)75 1739 y(MPI_DOUBLE_COMPLEX)75 1795 y(MPI_LOGICAL)75 1852 y(MPI_CHARACTER)75 1908 y(MPI_BYTE)75 2021 y(/*)i(reserved)e (communicators)g(\(C)i(and)f(Fortran\))g(*/)75 2077 y(MPI_COMM_WORLD)75 2190 y(/*)h(collective)e(operations)h(\(C)g(and)h(Fortran\))e(*/)75 2247 y(MPI_MAX)75 2303 y(MPI_MIN)75 2360 y(MPI_SUM)75 2416 y(MPI_PROD)75 2473 y(MPI_AND)75 2529 y(MPI_OR)75 2586 y(MPI_XOR)75 2642 y(MPI_MAXLOC)75 2698 y(MPI_MINLOC)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 177 185 177 184 bop 75 -100 a Fs(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b FC(177)75 102 y Fo(/*)24 b(topologies)e(\(C)i(and)f(Fortran\))g(*/)75 158 y(MPI_REORDER)75 214 y(MPI_NOREORDER)75 271 y(MPI_GRAPHTOP)75 327 y(MPI_CARTTOP)75 384 y(MPI_NOTOP)75 440 y(MPI_PER)75 497 y(MPI_NOPER)166 668 y FC(The)15 b(follo)o(wing)h(are)f(de\014ned)i (C)e(t)o(yp)q(e)g(de\014nitions,)i(also)e(included)j(in)e(the)f(\014le) h Fo(mpi.h)p FC(.)75 785 y Fo(/*)24 b(opaque)f(types)g(\(C\))g(*/)75 842 y(MPI_Persistence)75 898 y(MPI_Mode)75 955 y(MPI_Op)75 1011 y(MPI_Comm_name)75 1067 y(MPI_Aint)75 1124 y(MPI_Errmode)75 1237 y(/*)h(assorted)e(opaque)h(structures)g(\(C\))g(*/)75 1293 y(MPI_Status)75 1350 y(MPI_Group)75 1406 y(MPI_Communicator)75 1463 y(MPI_Datatype)75 1519 y(MPI_Comm_request)75 1576 y(MPI_Attribute)75 1632 y(MPI_Intercomm_request)75 1785 y Ft(A.3)59 b(C)20 b(bindings)g(fo)n(r)h(P)n(oint-to-P)n(oint)e (Communication)75 1890 y FC(These)d(are)e(presen)o(ted)i(here)g(in)g (the)f(order)g(of)g(their)g(app)q(earance)h(in)g(the)g(c)o(hapter.)75 1948 y Fo(int)23 b(MPI)p 245 1948 15 2 v 17 w(Send\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 1948 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 2005 y(int)h(tag,)f(MPI)p 683 2005 V 17 w(Comm)g(comm\))75 2093 y(int)g(MPI)p 245 2093 V 17 w(Recv\(void*)g(buf,)g(int)g(count,)h (MPI)p 979 2093 V 16 w(Datatype)f(datatype,)g(int)g(source,)393 2150 y(int)h(tag,)f(MPI)p 683 2150 V 17 w(Comm)g(comm,)g(MPI)p 1034 2150 V 17 w(Status)g(*status\))75 2238 y(int)g(MPI)p 245 2238 V 17 w(Get)p 334 2238 V 17 w(source\(MPI)p 591 2238 V 16 w(Status)g(status,)g(int)g(*source\))75 2326 y(int)g(MPI)p 245 2326 V 17 w(Get)p 334 2326 V 17 w(tag\(MPI)p 519 2326 V 16 w(Status)g(status,)g(int)h(*tag\))75 2414 y(int)f(MPI)p 245 2414 V 17 w(Get)p 334 2414 V 17 w(count\(MPI)p 567 2414 V 16 w(Status)g(status,)g(MPI)p 1013 2414 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 2502 y(int)f(MPI)p 245 2502 V 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2502 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2559 y(int)h(tag,)f(MPI)p 683 2559 V 17 w(Comm)g(comm\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(Ssend\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 2647 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2704 y(int)h(tag,)f(MPI)p 683 2704 V 17 w(Comm)g(comm\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 178 186 178 185 bop 75 -100 a FC(178)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fo(int)23 b(MPI)p 245 45 15 2 v 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 45 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 102 y(int)h(tag,)f(MPI)p 683 102 V 17 w(Comm)g(comm,)g(MPI)p 1034 102 V 17 w(Comm)p 1147 102 V 17 w(request)g(request\))75 191 y(int)g(MPI)p 245 191 V 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g (MPI)p 1026 191 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 247 y(int)g(tag,)f(MPI)p 683 247 V 17 w(Comm)g(comm,)g(MPI)p 1034 247 V 17 w(Comm)p 1147 247 V 17 w(request)g(request\))75 336 y(int)g(MPI)p 245 336 V 17 w(Issend\(void*)f(buf,)i(int)f(count,)g (MPI)p 1026 336 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 392 y(int)g(tag,)f(MPI)p 683 392 V 17 w(Comm)g(comm,)g(MPI)p 1034 392 V 17 w(Comm)p 1147 392 V 17 w(request)g(request\))75 481 y(int)g(MPI)p 245 481 V 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 481 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 538 y(int)h(tag,)f(MPI)p 683 538 V 17 w(Comm)g(comm,)g(MPI)p 1034 538 V 17 w(Comm)p 1147 538 V 17 w(request)g(request\))75 627 y(int)g(MPI)p 245 627 V 17 w(Wait\(MPI)p 454 627 V 16 w(Comm)p 566 627 V 17 w(request)g(*request,)f(MPI)p 1084 627 V 17 w(Status)h(*status\))75 716 y(int)g(MPI)p 245 716 V 17 w(Test\(MPI)p 454 716 V 16 w(Comm)p 566 716 V 17 w(request)g(*request,)f(int)i(flag,)f(MPI)p 1323 716 V 17 w(Status)g(*status\))75 805 y(int)g(MPI)p 245 805 V 17 w(Waitany\(int)g(count,)g(MPI)p 788 805 V 16 w(Comm)p 900 805 V 17 w(request)g(*array)p 1252 805 V 16 w(of)p 1316 805 V 17 w(requests,)g(int)g(*index,)393 861 y(MPI)p 468 861 V 17 w(Status)g(*status\))75 950 y(int)g(MPI)p 245 950 V 17 w(Testany\(int)g(count,)g(MPI)p 788 950 V 16 w(Comm)p 900 950 V 17 w(request)g(*array)p 1252 950 V 16 w(of)p 1316 950 V 17 w(requests,)g(int)g(*index,)393 1007 y(int)h(*flag,)f(MPI)p 731 1007 V 16 w(Status)g(*status\))75 1095 y(int)g(MPI)p 245 1095 V 17 w(Waitall\(int)g(count,)g(MPI)p 788 1095 V 16 w(Comm)p 900 1095 V 17 w(request)g(*array)p 1252 1095 V 16 w(of)p 1316 1095 V 17 w(requests,)393 1152 y(MPI)p 468 1152 V 17 w(Status)g(*array)p 796 1152 V 16 w(of)p 860 1152 V 17 w(statuses\))75 1241 y(int)g(MPI)p 245 1241 V 17 w(Testall\(int)g(count,)g(MPI)p 788 1241 V 16 w(Comm)p 900 1241 V 17 w(request)g(*array)p 1252 1241 V 16 w(of)p 1316 1241 V 17 w(requests,)g(int)g(*flag,)393 1297 y(MPI)p 468 1297 V 17 w(Status)g(*array)p 796 1297 V 16 w(of)p 860 1297 V 17 w(statuses\))75 1386 y(int)g(MPI)p 245 1386 V 17 w(Iprobe\(int)g(source,)g(int)g(tag,)g(MPI)p 1002 1386 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 1443 y(MPI)p 468 1443 V 17 w(Status)g(*status\))75 1532 y(int)g(MPI)p 245 1532 V 17 w(Probe\(int)g(source,)g(int)g(tag,)h(MPI)p 979 1532 V 16 w(Comm)g(comm,)f(MPI)p 1330 1532 V 17 w(Status)g (*status\))75 1621 y(int)g(MPI)p 245 1621 V 17 w(Cancel\(MPI)p 502 1621 V 16 w(Comm)p 614 1621 V 17 w(request)g(request\))75 1710 y(int)g(MPI)p 245 1710 V 17 w(Test)p 358 1710 V 17 w(cancelled\(MPI)p 687 1710 V 15 w(Status)g(status,)g(int)h(*flag\)) 75 1799 y(int)f(MPI)p 245 1799 V 17 w(Create)p 406 1799 V 17 w(send\(void*)f(buf,)h(int)h(count,)f(MPI)p 1139 1799 V 17 w(Datatype)f(datatype,)h(int)h(dest,)393 1855 y(int)g(tag,)f(MPI)p 683 1855 V 17 w(Comm)g(comm,)g(MPI)p 1034 1855 V 17 w(Comm)p 1147 1855 V 17 w(request)g(request\))75 1944 y(int)g(MPI)p 245 1944 V 17 w(Create)p 406 1944 V 17 w(rsend\(void*)f(buf,)h(int)h(count,)f(MPI)p 1163 1944 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 2000 y(int)h(tag,)f(MPI)p 683 2000 V 17 w(Comm)g(comm,)g(MPI)p 1034 2000 V 17 w(Comm)p 1147 2000 V 17 w(request)g(request\))75 2089 y(int)g(MPI)p 245 2089 V 17 w(Create)p 406 2089 V 17 w(ssend\(void*)f(buf,)h(int)h(count,)f(MPI)p 1163 2089 V 17 w(Datatype)f(datatype,)h(int)g(dest,)393 2146 y(int)h(tag,)f(MPI)p 683 2146 V 17 w(Comm)g(comm,)g(MPI)p 1034 2146 V 17 w(Comm)p 1147 2146 V 17 w(request)g(request\))75 2235 y(int)g(MPI)p 245 2235 V 17 w(Create)p 406 2235 V 17 w(recv\(void*)f(buf,)h(int)h(count,)f(MPI)p 1139 2235 V 17 w(Datatype)f(datatype,)h(int)h(source,)393 2291 y(int)g(tag,)f(MPI)p 683 2291 V 17 w(Comm)g(comm,)g(MPI)p 1034 2291 V 17 w(Comm)p 1147 2291 V 17 w(request)g(request\))75 2380 y(int)g(MPI)p 245 2380 V 17 w(Start\(MPI)p 478 2380 V 16 w(Comm)p 590 2380 V 17 w(request)g(request\))75 2469 y(int)g(MPI)p 245 2469 V 17 w(Startall\(int)f(count,)h(MPI)p 811 2469 V 17 w(Comm)p 924 2469 V 17 w(request)g(*array)p 1276 2469 V 16 w(of)p 1340 2469 V 17 w(requests\))75 2558 y(int)g(MPI)p 245 2558 V 17 w(Request)p 430 2558 V 16 w(free\(MPI)p 638 2558 V 17 w(Comm)p 751 2558 V 16 w(request)g(request\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(Sendrecv\(void)f(*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 2647 V 17 w(Datatype)f(sendtype,)393 2704 y(int)i(dest,)f(int)g (sendtag,)g(void)g(*recvbuf,)g(int)g(recvcount,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 179 187 179 186 bop 75 -100 a Fs(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b FC(179)393 45 y Fo(MPI)p 468 45 15 2 v 17 w(Datatype)23 b(recvtype,)f(int)i(source,)f (MPI)p 1297 45 V 16 w(Datatype)g(recvtag,)393 102 y(MPI)p 468 102 V 17 w(Comm)g(comm,)g(MPI)p 819 102 V 17 w(Status)g(*status\)) 75 188 y(int)g(MPI)p 245 188 V 17 w(Sendrecv)p 454 188 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 188 V 17 w(Datatype)g(datatype,)393 244 y(int)h(dest,)f(int)g(sendtag,) g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 244 V 16 w(Comm)h(comm,)393 301 y(MPI)p 468 301 V 17 w(Status)f(*status\))75 387 y(int)g(MPI)p 245 387 V 17 w(Type)p 358 387 V 17 w(contiguous\(int)f(count,)h(MPI)p 972 387 V 17 w(Datatype)f(oldtype,) 393 444 y(MPI)p 468 444 V 17 w(Datatype)h(*newtype\))75 530 y(int)g(MPI)p 245 530 V 17 w(Type)p 358 530 V 17 w(vector\(int)f(count,)h(int)h(blocklength,)e(int)i(stride,)393 587 y(MPI)p 468 587 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 587 V 17 w(Datatype)h(*newtype\))75 673 y(int)g(MPI)p 245 673 V 17 w(Type)p 358 673 V 17 w(hvector\(int)f(count,)h(int)h (blocklength,)e(MPI)p 1306 673 V 17 w(Aint)h(stride,)393 730 y(MPI)p 468 730 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 730 V 17 w(Datatype)h(*newtype\))75 816 y(int)g(MPI)p 245 816 V 17 w(Type)p 358 816 V 17 w(indexed\(int)f(count,)h(int)h (*array)p 1068 816 V 16 w(of)p 1132 816 V 17 w(blocklengths,)393 873 y(int)g(*array)p 636 873 V 16 w(of)p 700 873 V 17 w(displacements,)e(MPI)p 1147 873 V 17 w(Datatype)g(oldtype,)393 929 y(MPI)p 468 929 V 17 w(Datatype)h(*newtype\))75 1015 y(int)g(MPI)p 245 1015 V 17 w(Type)p 358 1015 V 17 w(hindexed\(int)f (count,)h(int)h(*array)p 1092 1015 V 16 w(of)p 1156 1015 V 17 w(blocklengths,)393 1072 y(MPI)p 468 1072 V 17 w(Aint)f(*array)p 748 1072 V 17 w(of)p 813 1072 V 17 w(displacements,)e(MPI)p 1259 1072 V 17 w(Datatype)i(oldtype,)393 1128 y(MPI)p 468 1128 V 17 w(Datatype)g(*newtype\))75 1215 y(int)g(MPI)p 245 1215 V 17 w(Type)p 358 1215 V 17 w(struct\(int)f(count,)h(int)h (*array)p 1044 1215 V 16 w(of)p 1108 1215 V 17 w(blocklengths,)393 1271 y(MPI)p 468 1271 V 17 w(Aint)f(*array)p 748 1271 V 17 w(of)p 813 1271 V 17 w(displacements,)e(MPI)p 1259 1271 V 17 w(Datatype)i(*array)p 1635 1271 V 16 w(of)p 1699 1271 V 17 w(types,)393 1328 y(MPI)p 468 1328 V 17 w(Datatype)g(*newtype\))75 1414 y(int)g(MPI)p 245 1414 V 17 w(Address\(void*)f(location,)h(MPI)p 907 1414 V 17 w(Aint)g(address\))75 1501 y(int)g(MPI)p 245 1501 V 17 w(Type)p 358 1501 V 17 w(extent\(MPI)p 615 1501 V 16 w(Datatype)f(datatype,)h(MPI)p 1156 1501 V 17 w(Aint)g(*extent\)) 75 1587 y(int)g(MPI)p 245 1587 V 17 w(Type)p 358 1587 V 17 w(size\(MPI)p 567 1587 V 16 w(Datatype)g(datatype,)f(MPI)p 1108 1587 V 17 w(Aint)h(*size\))75 1674 y(int)g(MPI)p 245 1674 V 17 w(Type)p 358 1674 V 17 w(count\(MPI)p 591 1674 V 16 w(Datatype)g(datatype,)f(int)i(*count\))75 1760 y(int)f(MPI)p 245 1760 V 17 w(Type)p 358 1760 V 17 w(commit\(MPI)p 615 1760 V 16 w(Datatype)f(*datatype\))75 1846 y(int)h(MPI)p 245 1846 V 17 w(Type)p 358 1846 V 17 w(free\(MPI)p 567 1846 V 16 w(Datatype)g(*datatype\))75 1933 y(int)g(MPI)p 245 1933 V 17 w(Get)p 334 1933 V 17 w(elements\(MPI)p 639 1933 V 16 w(Status)g(status,)f(MPI)p 1084 1933 V 17 w(Datatype)h(datatype,)g(int)g(*count\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 180 188 180 187 bop 75 -100 a FC(180)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Ft(A.4)59 b(C)20 b(Bindings)g(fo)n (r)h(Collective)e(Communication)75 149 y Fo(int)k(MPI)p 245 149 15 2 v 17 w(Barrier\(MPI)p 526 149 V 16 w(Comm)g(comm)h(\))75 236 y(int)47 b(MPI)p 269 236 V 17 w(Bcast\(void*)22 b(buf,)i(int)f (count,)g(MPI)p 1026 236 V 17 w(Datatype)g(datatype,)f(int)i(root,)393 293 y(MPI)p 468 293 V 17 w(Comm)f(comm)h(\))75 380 y(int)f(MPI)p 245 380 V 17 w(Gather\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1217 380 V 17 w(Datatype)h(sendtype,)393 437 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 437 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 493 y(MPI)p 468 493 V 17 w(Comm)f(comm\))75 581 y(int)g(MPI)p 245 581 V 17 w(Gatherv\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 581 V 17 w(Datatype)h(sendtype,)393 637 y(void*)g(recvbuf,)g(int)h (*recvcounts,)e(int)h(*displs,)393 694 y(MPI)p 468 694 V 17 w(Datatype)g(recvtype,)f(int)i(root,)f(MPI)p 1249 694 V 17 w(Comm)g(comm\))75 781 y(int)g(MPI)p 245 781 V 17 w(Scatter\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 781 V 17 w(Datatype)h(sendtype,)393 837 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 837 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 894 y(MPI)p 468 894 V 17 w(Comm)f(comm\))75 981 y(int)g(MPI)p 245 981 V 17 w(Scatterv\(void*)f(sendbuf,)h(int)g(*sendcounts,)g(int)g (*displs,)393 1038 y(MPI)p 468 1038 V 17 w(Datatype)g(sendtype,)f (void*)i(recvbuf,)e(int)i(recvcount,)393 1094 y(MPI)p 468 1094 V 17 w(Datatype)f(recvtype,)f(int)i(root,)f(MPI)p 1249 1094 V 17 w(Comm)g(comm\))75 1182 y(int)g(MPI)p 245 1182 V 17 w(Allgather\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1289 1182 V 17 w(Datatype)f(sendtype,)393 1238 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1238 V 17 w(Datatype)h(recvtype,)393 1295 y(MPI)p 468 1295 V 17 w(Comm)g(comm\))75 1382 y(int)g(MPI)p 245 1382 V 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1382 V 16 w(Datatype)g(sendtype,)393 1439 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1495 y(MPI)p 468 1495 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1495 V 17 w(Comm)h(comm\))75 1583 y(int)g(MPI)p 245 1583 V 17 w(Alltoall\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1583 V 17 w(Datatype)f(sendtype,)393 1639 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1639 V 17 w(Datatype)h(recvtype,)393 1696 y(MPI)p 468 1696 V 17 w(Comm)g(comm\))75 1783 y(int)g(MPI)p 245 1783 V 17 w(Alltoallv\(void*)f(sendbuf,)h(int)g(*sendcounts,)f(int) i(*sdispls,)393 1840 y(MPI)p 468 1840 V 17 w(Datatype)f(sendtype,)f (void*)i(recvbuf,)e(int)i(*recvcounts,)393 1896 y(int)g(*rdispls,)e (MPI)p 802 1896 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1896 V 17 w(Comm)i(comm\))75 1984 y(int)f(MPI)p 245 1984 V 17 w(Reduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)h(count,)393 2040 y(MPI)p 468 2040 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2040 V 17 w(Op)i(op,)f(int)h(root,)f(MPI)p 1505 2040 V 17 w(Comm)g(comm\))75 2128 y(int)g(MPI)p 245 2128 V 17 w(User)p 358 2128 V 17 w(reduce\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)g(count,)393 2184 y(MPI)p 468 2184 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(int)i(root,)f(MPI)p 1607 2184 V 17 w(Comm)g(comm\))75 2272 y(int)g(MPI)p 245 2272 V 17 w(User)p 358 2272 V 17 w(reducea\(void*)f(sendbuf,)h (void*)g(recvbuf,)g(int)g(count,)393 2328 y(MPI)p 468 2328 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(int)i(root,)f (MPI)p 1607 2328 V 17 w(Comm)g(comm\))75 2416 y(int)g(MPI)p 245 2416 V 17 w(Allreduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g (count,)393 2472 y(MPI)p 468 2472 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2472 V 17 w(Op)i(op,)f(MPI)p 1266 2472 V 17 w(Comm)g(comm\))75 2560 y(int)g(MPI)p 245 2560 V 17 w(User)p 358 2560 V 17 w(allreduce\(void*)f(sendbuf,)g(void*)i(recvbuf,)e(int)i(count,)393 2616 y(MPI)p 468 2616 V 17 w(Datatype)f(datatype,)f(void)i(function,)e (MPI)p 1368 2616 V 17 w(Comm)h(comm\))75 2704 y(int)g(MPI)p 245 2704 V 17 w(User)p 358 2704 V 17 w(allreducea\(void*)f(sendbuf,)g (void*)h(recvbuf,)g(int)h(count,)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 181 189 181 188 bop 75 -100 a Fs(A.5.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(GR)o (OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)160 b FC(181)393 45 y Fo(MPI)p 468 45 15 2 v 17 w(Datatype)23 b(datatype,)f(void)i (function,)e(MPI)p 1368 45 V 17 w(Comm)h(comm\))75 134 y(int)g(MPI)p 245 134 V 17 w(Reduce)p 406 134 V 17 w(scatter\(void*)f (sendbuf,)g(void*)i(recvbuf,)e(int)i(*recvcounts,)393 190 y(MPI)p 468 190 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 190 V 17 w(Op)i(op,)f(MPI)p 1266 190 V 17 w(Comm)g(comm\))75 279 y(int)g(MPI)p 245 279 V 17 w(Scan\(void*)g(sendbuf,)f(void*)i (recvbuf,)e(int)i(count,)393 335 y(MPI)p 468 335 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 335 V 17 w(Op)i(op,)f(MPI)p 1266 335 V 17 w(Comm)g(comm)h(\))75 424 y(int)f(MPI)p 245 424 V 17 w(User)p 358 424 V 17 w(scan\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)h(count,)393 480 y(MPI)p 468 480 V 17 w(Datatype)f(datatype,)f(void)i(function,)e(MPI)p 1368 480 V 17 w(Comm)h(comm\))75 569 y(int)g(MPI)p 245 569 V 17 w(User)p 358 569 V 17 w(scana\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)g(count,)393 625 y(MPI)p 468 625 V 17 w(Datatype)g(datatype,)f(void)i(function,)e(MPI)p 1368 625 V 17 w(Comm)h(comm\))75 810 y Ft(A.5)59 b(C)20 b(Bindings)g(fo)n(r) h(Groups,)e(Contexts,)f(and)h(Communicato)n(rs)75 916 y Fo(int)k(MPI)p 245 916 V 17 w(Group)p 382 916 V 17 w(size\(MPI)p 591 916 V 16 w(Group)g(group,)g(int)g(*size\))75 1004 y(int)g(MPI)p 245 1004 V 17 w(Group)p 382 1004 V 17 w(rank\(MPI)p 591 1004 V 16 w(Group)g(group,)g(int)g(*rank\))75 1093 y(int)g(MPI)p 245 1093 V 17 w(Group)p 382 1093 V 17 w(translate)p 615 1093 V 16 w(ranks)g(\(MPI)p 870 1093 V 16 w(Group)h(group1,)f(int)g(n,)h(int)f(*ranks1,)393 1149 y(MPI)p 468 1149 V 17 w(Group)g(group2,)g(int)g(*ranks2\))75 1238 y(int)g(MPI)p 245 1238 V 17 w(Group)p 382 1238 V 17 w(union\(MPI)p 615 1238 V 16 w(Group)g(group1,)g(MPI)p 1037 1238 V 17 w(Group)g(group2,)g(MPI)p 1460 1238 V 16 w(Group)h(*newgroup\))75 1327 y(int)f(MPI)p 245 1327 V 17 w(Group)p 382 1327 V 17 w(intersection\(MPI)p 783 1327 V 15 w(Group)g(group1,)g(MPI)p 1204 1327 V 17 w(Group)g(group2,) 393 1383 y(MPI)p 468 1383 V 17 w(Group)g(*newgroup\))75 1472 y(int)g(MPI)p 245 1472 V 17 w(Group)p 382 1472 V 17 w(difference\(MPI)p 735 1472 V 15 w(Group)g(group1,)g(MPI)p 1156 1472 V 17 w(Group)g(group2,)393 1528 y(MPI)p 468 1528 V 17 w(Group)g(*newgroup\))75 1617 y(int)g(MPI)p 245 1617 V 17 w(Group)p 382 1617 V 17 w(incl\(MPI)p 591 1617 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1617 V 17 w(Group)h(*newgroup\))75 1705 y(int)g(MPI)p 245 1705 V 17 w(Group)p 382 1705 V 17 w(excl\(MPI)p 591 1705 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1705 V 17 w(Group)h(*newgroup\))75 1794 y(int)g(MPI)p 245 1794 V 17 w(Group)p 382 1794 V 17 w(range)p 519 1794 V 16 w(incl\(MPI)p 727 1794 V 16 w(Group)g(group,)h(int)f(n,)h(int)f (**ranges,)393 1850 y(MPI)p 468 1850 V 17 w(Group)g(*newgroup\))75 1939 y(int)g(MPI)p 245 1939 V 17 w(Group)p 382 1939 V 17 w(range)p 519 1939 V 16 w(excl\(MPI)p 727 1939 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(**ranges,)393 1995 y(MPI)p 468 1995 V 17 w(Group)g(*newgroup\))75 2084 y(int)g(MPI)p 245 2084 V 17 w(Group)p 382 2084 V 17 w(free\(MPI)p 591 2084 V 16 w(Group)g(*group\))75 2172 y(int)g(MPI)p 245 2172 V 17 w(Comm)p 358 2172 V 17 w(size\(MPI)p 567 2172 V 16 w(Comm)g(comm,)g(int)h(*size\))75 2261 y(int)f(MPI)p 245 2261 V 17 w(Comm)p 358 2261 V 17 w(rank\(MPI)p 567 2261 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 2349 y(int)f(MPI)p 245 2349 V 17 w(Comm)p 358 2349 V 17 w(group\(MPI)p 591 2349 V 16 w(Comm)g(comm,)g(MPI)p 941 2349 V 17 w(Group)g(*group\))75 2438 y(int)g(MPI)p 245 2438 V 17 w(Comm)p 358 2438 V 17 w(dup\(MPI)p 543 2438 V 16 w(Comm)g(comm,)h(MPI)p 894 2438 V 16 w(Comm)g(*newcomm\))75 2526 y(int)f(MPI)p 245 2526 V 17 w(Comm)p 358 2526 V 17 w(make\(MPI)p 567 2526 V 16 w(Comm)g(comm,)g(MPI)p 917 2526 V 17 w(Group)g(group,)g(MPI)p 1316 2526 V 17 w(Comm)h(*newcomm\))75 2615 y(int)f(MPI)p 245 2615 V 17 w(Comm)p 358 2615 V 17 w(split\(MPI)p 591 2615 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 2615 V 16 w(Comm)g(*newcomm\))75 2704 y(int)f(MPI)p 245 2704 V 17 w(Comm)p 358 2704 V 17 w(free\(MPI)p 567 2704 V 16 w(Comm)g(*comm\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 182 190 182 189 bop 75 -100 a FC(182)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fo(int)23 b(MPI)p 245 45 15 2 v 17 w(Comm)p 358 45 V 17 w(test)p 471 45 V 16 w(inter\(MPI)p 703 45 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 133 y(int)g(MPI)p 245 133 V 17 w(Intercomm)p 478 133 V 16 w(start\(MPI)p 710 133 V 16 w(Comm)g(local)p 965 133 V 17 w(comm,)g(int)h(local)p 1341 133 V 16 w(leader,)393 189 y(MPI)p 468 189 V 17 w(Comm)f(peer)p 700 189 V 17 w(comm,)g(int)h(remote)p 1100 189 V 16 w(leader,)f(int)g(tag,)393 246 y(MPI)p 468 246 V 17 w(Intercomm)p 701 246 V 16 w(request)g(*inter) p 1052 246 V 16 w(request\))75 333 y(int)g(MPI)p 245 333 V 17 w(Intercomm)p 478 333 V 16 w(finish\(MPI)p 734 333 V 16 w(Intercomm)p 966 333 V 16 w(request)g(*inter)p 1317 333 V 16 w(request,)393 390 y(MPI)p 468 390 V 17 w(Comm)g(*newcomm\))75 477 y(int)g(MPI)p 245 477 V 17 w(Intercomm)p 478 477 V 16 w(merge\(MPI)p 710 477 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 534 y(MPI)p 468 534 V 17 w(Comm)g(*newintracomm\))75 622 y(int)g(MPI)p 245 622 V 17 w(Intercomm)p 478 622 V 16 w(name)p 590 622 V 17 w(start\(MPI)p 823 622 V 16 w(Comm)g(comm,)g(char)h(*name,)393 678 y(MPI)p 468 678 V 17 w(Intercomm)p 701 678 V 16 w(request)f(*inter) p 1052 678 V 16 w(request\))75 766 y(int)g(MPI)p 245 766 V 17 w(Attr)p 358 766 V 17 w(create)p 519 766 V 16 w(key\(int)g(copy)p 822 766 V 17 w(fn\(\),)g(int)g(delete)p 1221 766 V 17 w(fn\(\),)g(int)g(keyval,)393 822 y(void*)g(extra)p 659 822 V 17 w(state\))75 910 y(int)g(MPI)p 245 910 V 17 w(Attr)p 358 910 V 17 w(free)p 471 910 V 16 w(key\(int)g(keyval\))75 998 y(int)g(MPI)p 245 998 V 17 w(Put)p 334 998 V 17 w(attr\(MPI)p 543 998 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 998 V 16 w(val\))75 1085 y(int)g(MPI)p 245 1085 V 17 w(Get)p 334 1085 V 17 w(attr\(MPI)p 543 1085 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 1085 V 16 w(val,)g(int)h(*found\))75 1173 y(int)f(MPI)p 245 1173 V 17 w(Delete)p 406 1173 V 17 w(Attr\(MPI)p 615 1173 V 16 w(Comm)g(comm,)g(int)h(keyval\))75 1353 y Ft(A.6)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro)r(cess)e(T)-5 b(op)r(ologies)75 1457 y Fo(int)23 b(MPI)p 245 1457 V 17 w(Make)p 358 1457 V 17 w(cart\(MPI)p 567 1457 V 16 w(comm)g(comm)p 798 1457 V 17 w(old,)g(int)h(ndims,)f(int)g(*dims,)g (int)h(*periods,)393 1513 y(int)g(reorder,)e(MPI)p 778 1513 V 17 w(comm)h(*comm)p 1034 1513 V 17 w(cart\))75 1601 y(int)g(MPI)p 245 1601 V 17 w(Make)p 358 1601 V 17 w(dims\(int)g(nnodes,)g(int)g(ndims,)g(int)g(*dims\))75 1689 y(int)g(MPI)p 245 1689 V 17 w(Make)p 358 1689 V 17 w(graph\(MPI)p 591 1689 V 16 w(comm)g(comm)p 822 1689 V 17 w(old,)g(int)h(nnodes,)e(int)i(*index,)f(int)g(*edges,)393 1745 y(int)h(reorder,)e(MPI)p 778 1745 V 17 w(comm)h(*comm)p 1034 1745 V 17 w(graph\))75 1833 y(int)g(MPI)p 245 1833 V 17 w(Topo)p 358 1833 V 17 w(status\(MPI)p 615 1833 V 16 w(comm)g(comm,)g(int)h(*status\))75 1921 y(int)f(MPI)p 245 1921 V 17 w(Get)p 334 1921 V 17 w(graphdims\(MPI)p 663 1921 V 15 w(comm)h(comm,)f(int)g(*nnodes,)g(int)h(*nedges\))75 2008 y(int)f(MPI)p 245 2008 V 17 w(Get)p 334 2008 V 17 w(graph\(MPI)p 567 2008 V 16 w(comm)g(comm,)g(int)h(maxindex,)f(int)g (maxedges,)g(int)g(*index,)393 2065 y(int)h(*edges\))75 2152 y(int)f(MPI)p 245 2152 V 17 w(Get)p 334 2152 V 17 w(cartdim\(MPI)p 615 2152 V 16 w(comm)g(comm,)g(int)h(*ndims\))75 2240 y(int)f(MPI)p 245 2240 V 17 w(Get)p 334 2240 V 17 w(cart\(MPI)p 543 2240 V 16 w(comm)g(comm,)h(int)f(maxdims,)g(int)g (*dims,)g(int)h(*periods,)393 2296 y(int)g(*coords\))75 2384 y(int)f(MPI)p 245 2384 V 17 w(Cart)p 358 2384 V 17 w(rank\(MPI)p 567 2384 V 16 w(comm)g(comm,)g(int)h(*coords,)f(int)g (*rank\))75 2472 y(int)g(MPI)p 245 2472 V 17 w(Cart)p 358 2472 V 17 w(coords\(MPI)p 615 2472 V 16 w(comm)g(comm,)g(int)h (rank,)f(int)g(maxdims,)g(int)g(*coords\))75 2559 y(int)g(MPI)p 245 2559 V 17 w(Graph)p 382 2559 V 17 w(neighbors)p 615 2559 V 16 w(count\(MPI)p 847 2559 V 16 w(comm)g(comm,)g(int)h(rank,)f (int)g(*nneighbors\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(Graph)p 382 2647 V 17 w(neighbors\(MPI)p 711 2647 V 15 w(comm)g(comm,)h(int)f(rank,)g(int)h(*maxneighbors,)393 2704 y(int)g(*neighbors\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 183 191 183 190 bop 75 -100 a Fs(A.7.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(ENVIR)o (ONMENT)l(AL)g(INQUIR)l(Y)582 b FC(183)75 45 y Fo(int)23 b(MPI)p 245 45 15 2 v 17 w(Cart)p 358 45 V 17 w(shift\(MPI)p 591 45 V 16 w(comm)g(comm,)g(int)h(direction,)e(int)i(disp,)f(int)g (*rank)p 1681 45 V 17 w(source,)393 102 y(int)h(*rank)p 612 102 V 16 w(dest\))75 190 y(int)f(MPI)p 245 190 V 17 w(Cart)p 358 190 V 17 w(sub\(MPI)p 543 190 V 16 w(comm)g(comm,)h (int)f(*remain)p 1085 190 V 16 w(dims,)g(MPI)p 1316 190 V 17 w(comm)h(*comm)p 1573 190 V 16 w(new\))75 279 y(int)f(MPI)p 245 279 V 17 w(Map)p 334 279 V 17 w(cart\(MPI)p 543 279 V 16 w(comm)g(comm)p 774 279 V 17 w(old,)g(int)h(ndims,)f(int)g(*dims,) g(int)h(*period,)393 336 y(int)g(*newrank\))75 424 y(int)f(MPI)p 245 424 V 17 w(Map)p 334 424 V 17 w(graph\(MPI)p 567 424 V 16 w(comm)g(comm)p 798 424 V 17 w(old,)g(int)h(nnodes,)f(int)g (*index,)g(int)g(*edges,)393 481 y(int)h(*newrank\))75 667 y Ft(A.7)59 b(C)20 b(bindings)g(fo)n(r)h(Environmental)d(Inquiry)75 773 y Fo(int)23 b(MPI)p 245 773 V 17 w(Get)p 334 773 V 17 w(valid)p 471 773 V 16 w(tag)p 559 773 V 17 w(range\(int)g(*low,)g (int)h(*high\))75 862 y(int)f(MPI)p 245 862 V 17 w(Get)p 334 862 V 17 w(processor)p 567 862 V 16 w(name\(int)g(len,)g(char)g (*name\))75 951 y(int)g(MPI)p 245 951 V 17 w(Get)p 334 951 V 17 w(host\(MPI)p 543 951 V 16 w(Comm)g(comm,)h(int)f(*rank\))75 1040 y(int)g(MPI)p 245 1040 V 17 w(Get)p 334 1040 V 17 w(buffer)p 495 1040 V 16 w(params\(int)g(*numsend,)f(int)i(*sizesend,)e (int)i(*numrecv,)393 1096 y(int)g(*sizerecv,)e(int)i(*totalnumsend,)e (int)h(*totalsizesend,)393 1152 y(int)h(*totalnumrecv,)e(int)h (*totalsizerecv\))75 1241 y(int)g(MPI)p 245 1241 V 17 w(Suggest)p 430 1241 V 16 w(buffer)p 590 1241 V 17 w(params\(int)f (numsend,)h(int)h(sizesend,)e(int)i(numrecv,)393 1298 y(int)g(sizerecv,)e(int)i(totalnumsend,)e(int)h(totalsizesen,)393 1354 y(int)h(totalnumrecv,)e(int)h(totalsizerecv,)f(int)i(*flag\))75 1443 y(int)f(MPI)p 245 1443 V 17 w(User)p 358 1443 V 17 w(specifies)p 591 1443 V 16 w(buffer\(MPI)p 847 1443 V 16 w(Comm)g(comm,)g(void*)g(buffer,)g(int)h(size\))75 1532 y(int)f(MPI)p 245 1532 V 17 w(Get)p 334 1532 V 17 w(errormode\(MPI)p 663 1532 V 15 w(Comm)h(comm,)f(MPI)p 1013 1532 V 17 w(Errmode)g(*mode\))75 1620 y(int)g(MPI)p 245 1620 V 17 w(Set)p 334 1620 V 17 w(errormode\(MPI)p 663 1620 V 15 w(Comm)h(comm,)f(MPI)p 1013 1620 V 17 w(Errmode)g(mode\)) 75 1709 y(int)g(MPI)p 245 1709 V 17 w(Set)p 334 1709 V 17 w(errhandler\(MPI)p 687 1709 V 15 w(Comm)h(comm,)f(void)g (handler\))75 1798 y(int)g(MPI)p 245 1798 V 17 w(Error)p 382 1798 V 17 w(string\(int)f(errorcode,)h(int)g(len,)g(char)h (*string\))75 1887 y(int)f(MPI)p 245 1887 V 17 w(Init\(int)g(*argc,)g (char)g(**argv\))75 1975 y(int)g(MPI)p 245 1975 V 17 w(Finalize\(\))75 2064 y(int)g(MPI)p 245 2064 V 17 w(Initialized\(int)f (*flag\))75 2153 y(int)h(MPI)p 245 2153 V 17 w(Abort\(MPI)p 478 2153 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 2339 y Ft(A.8)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(P)n(oint-to-P)n (oint)f(Communication)75 2445 y Fo(MPI)p 150 2445 V 17 w(SEND\(BUF,)i(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR\))170 2502 y()g(BUF\(*\))170 2558 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(IERROR)75 2647 y(MPI)p 150 2647 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS,)g(IERROR\))170 2704 y()g(BUF\(*\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 184 192 184 191 bop 75 -100 a FC(184)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fo(INTEGER)23 b(COUNT,)g(DATATYPE,) g(SOURCE,)g(TAG,)g(COMM,)g(STATUS\(MPI)p 1463 45 15 2 v 16 w(STATUS)p 1623 45 V 17 w(SIZE\),)170 102 y(IERROR)75 191 y(MPI)p 150 191 V 17 w(GET)p 239 191 V 17 w(SOURCE\(STATUS,)e (SOURCE,)i(IERROR\))170 247 y(INTEGER)g(STATUS\(MPI)p 604 247 V 16 w(STATUS)p 764 247 V 17 w(SIZE\),)g(SOURCE,)f(IERROR)75 336 y(MPI)p 150 336 V 17 w(GET)p 239 336 V 17 w(TAG\(STATUS,)g(TAG,)h (IERROR\))170 393 y(INTEGER)g(STATUS\(MPI)p 604 393 V 16 w(STATUS)p 764 393 V 17 w(SIZE\),)g(TAG,)g(IERROR)75 482 y(MPI)p 150 482 V 17 w(GET)p 239 482 V 17 w(COUNT\(STATUS,)f (DATATYPE,)g(COUNT,)h(IERROR\))170 538 y(INTEGER)g(STATUS\(MPI)p 604 538 V 16 w(STATUS)p 764 538 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)75 627 y(MPI)p 150 627 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,) g(DEST,)g(TAG,)g(COMM,)h(IERROR\))170 684 y()f(BUF\(*\))170 740 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 829 y(MPI)p 150 829 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 886 y()f(BUF\(*\))170 942 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 1031 y(MPI)p 150 1031 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(REQUEST,)e(IERROR\))170 1088 y()h(BUF\(*\))170 1144 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1233 y(MPI)p 150 1233 V 17 w(IRSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1290 y()g(BUF\(*\))170 1346 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1435 y(MPI)p 150 1435 V 17 w(ISSEND\(BUF,)g(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 1492 y()g(BUF\(*\))170 1548 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1637 y(MPI)p 150 1637 V 17 w(IRECV\(BUF,)g(COUNT,)h (DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1694 y()g(BUF\(*\))170 1750 y(INTEGER)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1839 y(MPI)p 150 1839 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 1896 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1896 V 16 w(STATUS)p 979 1896 V 16 w(SIZE\),)g(IERROR)75 1985 y(MPI)p 150 1985 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 2041 y(LOGICAL)g(FLAG)170 2098 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 2098 V 16 w(STATUS)p 979 2098 V 16 w(SIZE\),)g(IERROR)75 2187 y(MPI)p 150 2187 V 17 w(WAITANY\(COUNT,)f(ARRAY)p 645 2187 V 16 w(OF)p 709 2187 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g (IERROR\))170 2243 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2243 V 17 w(OF)p 716 2243 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 2243 V 16 w(STATUS)p 1610 2243 V 16 w(SIZE\),)170 2300 y(IERROR)75 2389 y(MPI)p 150 2389 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 2389 V 16 w(OF)p 709 2389 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))170 2445 y(LOGICAL)g(FLAG)170 2502 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2502 V 17 w(OF)p 716 2502 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 2502 V 16 w(STATUS)p 1610 2502 V 16 w(SIZE\),)170 2558 y(IERROR)75 2647 y(MPI)p 150 2647 V 17 w(WAITALL\(COUNT,)f(ARRAY)p 645 2647 V 16 w(OF)p 709 2647 V 17 w(REQUESTS,)h(ARRAY)p 1085 2647 V 16 w(OF)p 1149 2647 V 17 w(STATUSES,)g(IERROR\))170 2704 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2704 V 17 w(OF)p 716 2704 V 17 w(REQUESTS\(*\),)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 185 193 185 192 bop 75 -100 a Fs(A.8.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(POINT-TO-POINT)h(COMMUNICA)l(TION)185 b FC(185)170 45 y Fo(ARRAY)p 293 45 15 2 v 17 w(OF)p 358 45 V 17 w(STATUSES\(MPI)p 663 45 V 15 w(STATUS)p 822 45 V 17 w(SIZE,*\),)23 b(IERROR)75 134 y(MPI)p 150 134 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 134 V 16 w(OF)p 709 134 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 134 V 16 w(OF)p 1292 134 V 17 w(STATUSES,)g(IERROR\))170 191 y(LOGICAL)g(FLAG)170 247 y(INTEGER)g(COUNT,)g(ARRAY)p 651 247 V 17 w(OF)p 716 247 V 17 w(REQUESTS\(*\),)170 304 y(ARRAY)p 293 304 V 17 w(OF)p 358 304 V 17 w(STATUSES\(MPI)p 663 304 V 15 w(STATUS)p 822 304 V 17 w(SIZE,*\),)g(IERROR)75 393 y(MPI)p 150 393 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 449 y(LOGICAL)g(FLAG)170 506 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 506 V 16 w(STATUS)p 1218 506 V 16 w(SIZE\),)g(IERROR)75 595 y(MPI)p 150 595 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g (IERROR\))170 651 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 651 V 16 w(STATUS)p 1218 651 V 16 w(SIZE\),)g(IERROR)75 740 y(MPI)p 150 740 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 797 y(INTEGER)h(REQUEST,)g(IERROR)75 886 y(MPI)p 150 886 V 17 w(TEST)p 263 886 V 16 w(CANCELLED\(STATUS,)f(FLAG,)h(IERROR\)) 170 942 y(LOGICAL)g(FLAG)170 999 y(INTEGER)g(STATUS\(MPI)p 604 999 V 16 w(STATUS)p 764 999 V 17 w(SIZE\),)g(IERROR)75 1088 y(MPI)p 150 1088 V 17 w(CREATE)p 311 1088 V 16 w(SEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)i(TAG,)f(COMM,)g(REQUEST,)g(IERROR\))170 1144 y()g(BUF\(*\))170 1201 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1290 y(MPI)p 150 1290 V 17 w(CREATE)p 311 1290 V 16 w(RSEND\(BUF,)g (COUNT,)g(DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1346 y()g(BUF\(*\))170 1403 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1492 y(MPI)p 150 1492 V 17 w(CREATE)p 311 1492 V 16 w(SSEND\(BUF,)h(COUNT,)g (DATATYPE,)f(DEST,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1548 y()g(BUF\(*\))170 1605 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1694 y(MPI)p 150 1694 V 17 w(CREATE)p 311 1694 V 16 w(RECV\(BUF,)h(COUNT,)g (DATATYPE,)f(SOURCE,)h(TAG,)h(COMM,)f(REQUEST,)g(IERROR\))170 1750 y()g(BUF\(*\))170 1807 y(INTEGER)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1896 y(MPI)p 150 1896 V 17 w(START\(REQUEST,)f(IERROR\))170 1952 y(INTEGER)h (REQUEST,)g(IERROR)75 2041 y(MPI)p 150 2041 V 17 w(STARTALL\(COUNT,)f (ARRAY)p 669 2041 V 16 w(OF)p 733 2041 V 17 w(REQUESTS,)h(IERROR\))170 2098 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2098 V 17 w(OF)p 716 2098 V 17 w(REQUESTS\(*\),)f(IERROR)75 2187 y(MPI)p 150 2187 V 17 w(REQUEST)p 335 2187 V 16 w(FREE\(REQUEST,)g(IERROR\))170 2243 y(INTEGER)h(REQUEST,)g(IERROR)75 2332 y(MPI)p 150 2332 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(DEST,)i (SENDTAG,)e(RECVBUF,)393 2389 y(RECVCOUNT,)h(RECVTYPE,)f(SOURCE,)h (RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 2445 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 2502 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h(RECVTYPE,)170 2558 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 2558 V 16 w(STATUS)p 1122 2558 V 16 w(SIZE\),)h(IERROR)75 2647 y(MPI)p 150 2647 V 17 w(SENDRECV)p 359 2647 V 16 w(REPLACE\(BUF,)e(COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g (RECVTAG,)393 2704 y(COMM,)g(STATUS,)g(IERROR\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 186 194 186 193 bop 75 -100 a FC(186)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fo()23 b(BUF\(*\))170 102 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g (RECVTAG,)g(COMM,)170 158 y(STATUS\(MPI)p 413 158 15 2 v 16 w(STATUS)p 573 158 V 17 w(SIZE\),)g(IERROR)75 245 y(MPI)p 150 245 V 17 w(TYPE)p 263 245 V 16 w(CONTIGUOUS\(COUNT,)f (OLDTYPE,)h(NEWTYPE,)f(IERROR\))170 302 y(INTEGER)h(COUNT,)g(OLDTYPE,)g (NEWTYPE,)g(IERROR)75 389 y(MPI)p 150 389 V 17 w(TYPE)p 263 389 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)g(OLDTYPE,)f (NEWTYPE,)h(IERROR\))170 446 y(INTEGER)g(COUNT,)g(BLOCKLENGTH,)g (STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 533 y(MPI)p 150 533 V 17 w(TYPE)p 263 533 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)f(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 589 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 677 y(MPI)p 150 677 V 17 w(TYPE)p 263 677 V 16 w(INDEXED\(COUNT,)f (ARRAY)p 757 677 V 17 w(OF)p 822 677 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 677 V 16 w(OF)p 1357 677 V 17 w(DISPLACEMENTS,)393 733 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 790 y(INTEGER)g(COUNT,)g (ARRAY)p 651 790 V 17 w(OF)p 716 790 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY) p 1258 790 V 17 w(OF)p 1323 790 V 17 w(DISPLACEMENTS\(*\),)170 846 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 933 y(MPI)p 150 933 V 17 w(TYPE)p 263 933 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 933 V 17 w(OF)p 846 933 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 933 V 16 w(OF)p 1381 933 V 17 w(DISPLACEMENTS,)393 990 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1046 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1046 V 17 w(OF)p 716 1046 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1046 V 17 w(OF)p 1323 1046 V 17 w(DISPLACEMENTS\(*\),)170 1103 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1190 y(MPI)p 150 1190 V 17 w(TYPE)p 263 1190 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 1190 V 17 w(OF)p 798 1190 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 1190 V 16 w(OF)p 1333 1190 V 17 w(DISPLACEMENTS,)393 1246 y(ARRAY)p 516 1246 V 17 w(OF)p 581 1246 V 17 w(TYPES,)h(NEWTYPE,)f (IERROR\))170 1303 y(INTEGER)h(COUNT,)g(ARRAY)p 651 1303 V 17 w(OF)p 716 1303 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 1303 V 17 w(OF)p 1323 1303 V 17 w(DISPLACEMENTS\(*\),)170 1359 y(ARRAY)p 293 1359 V 17 w(OF)p 358 1359 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)75 1447 y(MPI)p 150 1447 V 17 w(ADDRESS\(LOCATION,)e (ADDRESS,)i(IERROR\))170 1503 y()g(LOCATION\(*\))170 1560 y(INTEGER)g(ADDRESS,)g(IERROR)75 1647 y(MPI)p 150 1647 V 17 w(TYPE)p 263 1647 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h (IERROR\))170 1703 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)75 1791 y(MPI)p 150 1791 V 17 w(TYPE)p 263 1791 V 16 w(SIZE\(DATATYPE,)f (SIZE,)i(IERROR\))170 1847 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)75 1935 y(MPI)p 150 1935 V 17 w(TYPE)p 263 1935 V 16 w(COUNT\(DATATYPE,)f (COUNT,)h(IERROR\))170 1991 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)75 2078 y(MPI)p 150 2078 V 17 w(TYPE)p 263 2078 V 16 w(COMMIT\(DATATYPE,)f (IERROR\))170 2135 y(INTEGER)h(DATATYPE,)g(IERROR)75 2222 y(MPI)p 150 2222 V 17 w(TYPE)p 263 2222 V 16 w(FREE\(DATATYPE,)f (IERROR\))170 2279 y(INTEGER)h(DATATYPE,)g(IERROR)75 2366 y(MPI)p 150 2366 V 17 w(GET)p 239 2366 V 17 w(ELEMENTS\(STATUS,)e (DATATYPE,)i(COUNT,)g(IERROR\))170 2422 y(INTEGER)g(STATUS\(MPI)p 604 2422 V 16 w(STATUS)p 764 2422 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)75 2600 y Ft(A.9)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i (Collective)e(Communication)75 2704 y Fo(MPI)p 150 2704 V 17 w(BARRIER\(COMM,)j(IERROR\))1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 187 195 187 194 bop 75 -100 a Fs(A.9.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(COLLECTIVE)g(COMMUNICA)l(TION)280 b FC(187)170 45 y Fo(INTEGER)23 b(COMM,)h(IERROR)75 136 y(MPI)p 150 136 15 2 v 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,)g(COMM,)g (IERROR\))170 192 y()g(BUFFER\(*\))170 249 y(INTEGER)g(COUNT,)g (DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 340 y(MPI)p 150 340 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 396 y(ROOT,)g(COMM,)h(IERROR\))170 453 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 509 y(INTEGER)g (SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g(COMM,)g (IERROR)75 600 y(MPI)p 150 600 V 17 w(GATHERV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)g(RECVBUF,)f(RECVCOUNTS,)h(DISPLS,)393 656 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 713 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 769 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g(ROOT,)170 826 y(COMM,)h(IERROR)75 917 y(MPI)p 150 917 V 17 w(SCATTER\(SENDBUF,)d (SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f(RECVCOUNT,)h(RECVTYPE,)393 973 y(ROOT,)g(COMM,)h(IERROR\))170 1029 y()f(SENDBUF\(*\),)g (RECVBUF\(*\))170 1086 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,) h(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR)75 1177 y(MPI)p 150 1177 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i(DISPLS,)g(SENDTYPE,) f(RECVBUF,)h(RECVCOUNT,)393 1233 y(RECVTYPE,)g(ROOT,)g(COMM,)g (IERROR\))170 1290 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1346 y(INTEGER)g(SENDCOUNTS\(*\),)f(DISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 1403 y(COMM,)h(IERROR)75 1493 y(MPI)p 150 1493 V 17 w(ALLGATHER\(SENDBUF,)d(SENDCOUNT,)i (SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 1550 y(COMM,)g(IERROR\))170 1606 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1663 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (COMM,)g(IERROR)75 1754 y(MPI)p 150 1754 V 17 w(ALLGATHERV\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNTS,)f(DISPLS,)393 1810 y(RECVTYPE,)h(COMM,)g(IERROR\))170 1867 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1923 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f (RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g(COMM,)170 1979 y(IERROR)75 2070 y(MPI)p 150 2070 V 17 w(ALLTOALL\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 2127 y(COMM,)g(IERROR\))170 2183 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2240 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,) h(RECVTYPE,)g(COMM,)g(IERROR)75 2330 y(MPI)p 150 2330 V 17 w(ALLTOALLV\(SENDBUF,)e(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h (RECVBUF,)g(RECVCOUNTS,)393 2387 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g (IERROR\))170 2443 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2500 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 2556 y(RECVTYPE,)h(COMM,)g (IERROR)75 2647 y(MPI)p 150 2647 V 17 w(REDUCE\(SENDBUF,)f(RECVBUF,)g (COUNT,)h(DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR\))170 2704 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 188 196 188 195 bop 75 -100 a FC(188)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fo(INTEGER)23 b(COUNT,)g(DATATYPE,) g(OP,)g(ROOT,)h(COMM,)f(IERROR)75 155 y(MPI)p 150 155 15 2 v 17 w(USER)p 263 155 V 16 w(REDUCE\(SENDBUF,)f(RECVBUF,)h(COUNT,) g(DATATYPE,)g(FUNCTION,)f(ROOT,)h(COMM,)393 211 y(IERROR\))170 267 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 324 y(EXTERNAL)g (FUNCTION)170 380 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g (IERROR)75 490 y(MPI)p 150 490 V 17 w(USER)p 263 490 V 16 w(REDUCEA\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)f (ROOT,)h(COMM,)393 546 y(IERROR\))170 603 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 659 y(EXTERNAL)g(FUNCTION)170 716 y(INTEGER)g(COUNT,) g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 825 y(MPI)p 150 825 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g(DATATYPE,)g(OP,)g (COMM,)g(IERROR\))170 882 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 938 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h(IERROR)75 1047 y(MPI)p 150 1047 V 17 w(USER)p 263 1047 V 16 w (ALLREDUCE\(SENDBUF,)e(RECVBUF,)h(COUNT,)g(DATATYPE,)f(FUNCTION,)h (COMM,)393 1104 y(IERROR\))170 1160 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1217 y(EXTERNAL)g(FUNCTION)170 1273 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g(IERROR)75 1383 y(MPI)p 150 1383 V 17 w(USER)p 263 1383 V 16 w(ALLREDUCEA\(SENDBUF,)f (RECVBUF,)g(COUNT,)h(DATATYPE,)g(FUNCTION,)g(COMM,)393 1439 y(IERROR\))170 1496 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1552 y(EXTERNAL)g(FUNCTION)170 1609 y(INTEGER)g(COUNT,)g(DATATYPE,)g (COMM,)g(IERROR)75 1718 y(MPI)p 150 1718 V 17 w(REDUCE)p 311 1718 V 16 w(SCATTER\(SENDBUF,)f(RECVBUF,)h(RECVCOUNTS,)f(DATATYPE,) h(OP,)g(COMM,)393 1774 y(IERROR\))170 1831 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1887 y(INTEGER)g(RECVCOUNTS,)g(DATATYPE,)f(OP,)i (COMM,)f(IERROR)75 1997 y(MPI)p 150 1997 V 17 w(SCAN\(SENDBUF,)f (RECVBUF,)h(COUNT,)g(DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 2053 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2110 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h(IERROR)75 2219 y(MPI)p 150 2219 V 17 w(USER)p 263 2219 V 16 w(SCAN\(SENDBUF,)e (RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)g(COMM,)g(IERROR\))170 2276 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2332 y(EXTERNAL)g(FUNCTION)170 2388 y(INTEGER)g(COUNT,)g(DATATYPE,)g(COMM,)g (IERROR)75 2498 y(MPI)p 150 2498 V 17 w(USER)p 263 2498 V 16 w(SCANA\(SENDBUF,)f(RECVBUF,)h(COUNT,)g(DATATYPE,)g(FUNCTION,)f (COMM,)i(IERROR\))170 2554 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2611 y(EXTERNAL)g(FUNCTION)170 2667 y(INTEGER)g(COUNT,)g(DATATYPE,)g (COMM,)g(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 189 197 189 196 bop 75 -100 a Fs(A.10.)27 b(F)o(OR)l(TRAN)11 b(BINDINGS)g(F)o(OR)g(GR)o(OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)p FC(189)75 45 y Ft(A.10)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i (Groups,)f(Contexts,)d(and)j(Communicato)n(rs)75 154 y Fo(MPI)p 150 154 15 2 v 17 w(GROUP)p 287 154 V 16 w(SIZE\(GROUP,)j (SIZE,)g(IERROR\))170 210 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 301 y(MPI)p 150 301 V 17 w(GROUP)p 287 301 V 16 w(RANK\(GROUP,)f(RANK,) g(IERROR\))170 357 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)75 447 y(MPI)p 150 447 V 17 w(GROUP)p 287 447 V 16 w(TRANSLATE)p 519 447 V 16 w(RANKS\(GROUP1,)e(N,)i(RANKS1,)f(GROUP2,)g(RANKS2,)g (IERROR\))170 504 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1\(*\),)e(GROUP2,)h (RANKS2\(*\),)g(IERROR)75 594 y(MPI)p 150 594 V 17 w(GROUP)p 287 594 V 16 w(UNION\(GROUP1,)f(GROUP2,)h(NEWGROUP,)g(IERROR\))170 650 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 741 y(MPI)p 150 741 V 17 w(GROUP)p 287 741 V 16 w (INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 797 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 887 y(MPI)p 150 887 V 17 w(GROUP)p 287 887 V 16 w(DIFFERENCE\(GROUP1,)f (GROUP2,)h(NEWGROUP,)f(IERROR\))170 944 y(INTEGER)h(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 1034 y(MPI)p 150 1034 V 17 w(GROUP)p 287 1034 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g(NEWGROUP,)g(IERROR\))170 1090 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f(NEWGROUP,)f(IERROR)75 1181 y(MPI)p 150 1181 V 17 w(GROUP)p 287 1181 V 16 w(EXCL\(GROUP,)h(N,) g(RANKS,)g(NEWGROUP,)g(IERROR\))170 1237 y(INTEGER)g(GROUP,)g(N,)h (RANKS\(*\),)f(NEWGROUP,)f(IERROR)75 1327 y(MPI)p 150 1327 V 17 w(GROUP)p 287 1327 V 16 w(RANGE)p 423 1327 V 17 w(INCL\(GROUP,)g(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1384 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 1474 y(MPI)p 150 1474 V 17 w(GROUP)p 287 1474 V 16 w(RANGE)p 423 1474 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1530 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(*,*\),)e(NEWGROUP,)h(IERROR)75 1620 y(MPI)p 150 1620 V 17 w(GROUP)p 287 1620 V 16 w(FREE\(GROUP,)g (IERROR\))170 1677 y(INTEGER)g(GROUP,)g(IERROR)75 1767 y(MPI)p 150 1767 V 17 w(COMM)p 263 1767 V 16 w(SIZE\(COMM,)g(SIZE,)g (IERROR\))170 1824 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 1914 y(MPI)p 150 1914 V 17 w(COMM)p 263 1914 V 16 w(RANK\(COMM,)g (RANK,)g(IERROR\))170 1970 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 2060 y(MPI)p 150 2060 V 17 w(COMM)p 263 2060 V 16 w(GROUP\(COMM,)g (GROUP,)g(IERROR\))170 2117 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)75 2207 y(MPI)p 150 2207 V 17 w(COMM)p 263 2207 V 16 w(DUP\(COMM,)g (NEWCOMM,)g(IERROR\))170 2264 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)75 2354 y(MPI)p 150 2354 V 17 w(COMM)p 263 2354 V 16 w(MAKE\(COMM,)h (GROUP,)g(NEWCOMM,)g(IERROR\))170 2410 y(INTEGER)g(COMM,)h(GROUP,)f (NEWCOMM,)f(IERROR)75 2500 y(MPI)p 150 2500 V 17 w(COMM)p 263 2500 V 16 w(SPLIT\(COMM,)h(COLOR,)g(KEY,)g(NEWCOMM,)g(IERROR\))170 2557 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g(NEWCOMM,)g(IERROR)75 2647 y(MPI)p 150 2647 V 17 w(COMM)p 263 2647 V 16 w(FREE\(COMM,)g (IERROR\))170 2704 y(INTEGER)g(COMM,)h(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 190 198 190 197 bop 75 -100 a FC(190)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(TEST)p 375 45 V 17 w(INTER\(COMM,)22 b(FLAG,)i(IERROR\))170 102 y(INTEGER)f(COMM,)h(IERROR)170 158 y(LOGICAL)f(FLAG)75 247 y(MPI)p 150 247 V 17 w(INTERCOMM)p 383 247 V 16 w(START\(LOCAL)p 663 247 V 15 w(COMM,)h(LOCAL)p 942 247 V 16 w(LEADER,)f(PEER)p 1245 247 V 17 w(COMM,)g(REMOTE)p 1549 247 V 16 w(LEADER,)g(TAG,)393 303 y(INTER)p 516 303 V 17 w(REQUEST,)f(IERROR\))170 360 y(INTEGER)h(LOCAL)p 484 360 V 17 w(COMM,)g(LOCAL)p 764 360 V 17 w(LEADER,)f(PEER)p 1067 360 V 17 w(COMM,)h(REMOTE)p 1371 360 V 17 w(LEADER,)g(TAG,)170 416 y(INTER)p 293 416 V 17 w(REQUEST,)g(IERROR)75 505 y(MPI)p 150 505 V 17 w(INTERCOMM)p 383 505 V 16 w(FINISH\(INTER)p 687 505 V 15 w(REQUEST,)g(IERROR\))170 562 y(INTEGER)g(INTER)p 484 562 V 17 w(REQUEST,)g(NEWCOMM,)f(IERROR)75 651 y(MPI)p 150 651 V 17 w(INTERCOMM)p 383 651 V 16 w(MERGE\(INTERCOMM,)f(HIGH,)j (NEWINTERCOMM,)e(IERROR\))170 707 y(INTEGER)h(INTERCOMM,)g (NEWINTRACOMM,)f(IERROR)170 763 y(LOGICAL)h(HIGH)75 852 y(MPI)p 150 852 V 17 w(INTERCOMM)p 383 852 V 16 w(NAME)p 495 852 V 16 w(START\(COMM,)g(NAME,)g(NEW)p 1013 852 V 17 w(COMM,)g(INTER)p 1293 852 V 16 w(REQUEST,)g(IERROR\))170 909 y(CHARACTER)g(*\(*\))g(NAME)170 965 y(INTEGER)g(COMM,)h(NEW)p 580 965 V 16 w(COMM,)f(INTER)p 859 965 V 17 w(REQUEST,)g(IERROR)75 1054 y(MPI)p 150 1054 V 17 w(ATTR)p 263 1054 V 16 w(CREATE)p 423 1054 V 17 w(KEY\(COPY)p 632 1054 V 16 w(FN,)g(DELETE)p 887 1054 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1310 1054 V 17 w(STATE,)g(IERROR\))170 1111 y(EXTERNAL)g(COPY)p 484 1111 V 17 w(FN,)g(DELETE)p 740 1111 V 17 w(FN)170 1167 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 1167 V 17 w(STATE,)g(IERROR)75 1256 y(MPI)p 150 1256 V 17 w(ATTR)p 263 1256 V 16 w(FREE)p 375 1256 V 17 w(KEY\(KEYVAL,)f(IERROR\))170 1312 y(INTEGER)h(KEYVAL,)g (IERROR)75 1401 y(MPI)p 150 1401 V 17 w(PUT)p 239 1401 V 17 w(ATTR\(COMM,)f(KEYVAL,)h(ATTRIBUTE)p 925 1401 V 16 w(VAL,)g(IERROR\))170 1458 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 1458 V 16 w(VAL,)i(IERROR)75 1547 y(MPI)p 150 1547 V 17 w(GET)p 239 1547 V 17 w(ATTR\(COMM,)e(KEYVAL,)h(ATTRIBUTE)p 925 1547 V 16 w(VAL,)g(FOUND,)g(IERROR\))170 1603 y(INTEGER)g(COMM,)h (KEYVAL,)e(ATTRIBUTE)p 914 1603 V 16 w(VAL,)i(IERROR)170 1660 y(LOGICAL)f(FOUND)75 1748 y(MPI)p 150 1748 V 17 w(DELETE)p 311 1748 V 16 w(ATTR\(COMM,)g(KEYVAL,)g(IERROR\))170 1805 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)75 1992 y Ft(A.11)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro)r(cess)e(T)-5 b(op)r(ologies)75 2098 y Fo(MPI)p 150 2098 V 17 w(MAKE)p 263 2098 V 16 w(CART\(COMM)p 495 2098 V 16 w(OLD,)24 b(NDIMS,)f(DIMS,)g(PERIODS,)g(REORDER,)f(COMM)p 1466 2098 V 17 w(CART,)h(IERROR\))170 2155 y(INTEGER)g(COMM)p 460 2155 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(PERIODS\(*\),)f(REORDER,)h (COMM)p 1575 2155 V 17 w(CART,)170 2211 y(IERROR)75 2300 y(MPI)p 150 2300 V 17 w(MAKE)p 263 2300 V 16 w(DIMS\(NNODES,)g(NDIMS,)g (DIMS,)g(IERROR\))170 2356 y(INTEGER)g(NNODES,)g(NDIMS,)g(DIMS\(*\),)g (IERROR)75 2445 y(MPI)p 150 2445 V 17 w(MAKE)p 263 2445 V 16 w(GRAPH\(COMM)p 519 2445 V 16 w(OLD,)h(NNODES,)e(INDEX,)h(EDGES,)h (REORDER,)e(COMM)p 1490 2445 V 17 w(GRAPH,)h(IERROR\))170 2502 y(INTEGER)g(COMM)p 460 2502 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g (EDGES\(*\),)f(REORDER,)h(COMM)p 1575 2502 V 17 w(GRAPH,)170 2558 y(IERROR)75 2647 y(MPI)p 150 2647 V 17 w(TOPO)p 263 2647 V 16 w(STATUS\(COMM,)g(STATUS,)g(IERROR\))170 2704 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 191 199 191 198 bop 75 -100 a Fs(A.12.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(ENVIR)o(ONMENT)l(AL)g(INQUIR)l(Y)364 b FC(191)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(GET)p 239 45 V 17 w(GRAPHDIMS\(COMM,)21 b(NNODES,)i(NEDGES,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(NNODES,)e(NEDGES,)h(IERROR)75 193 y(MPI)p 150 193 V 17 w(GET)p 239 193 V 17 w(GRAPH\(COMM,)f (MAXINDEX,)h(MAXEDGES,)f(INDEX,)h(EDGES,)g(IERROR\))170 249 y(INTEGER)g(COMM,)h(MAXINDEX,)e(MAXEDGES,)h(INDEX\(*\),)f (EDGES\(*\),)h(IERROR)75 340 y(MPI)p 150 340 V 17 w(GET)p 239 340 V 17 w(CARTDIM\(COMM,)f(NDIMS,)h(IERROR\))170 397 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)75 488 y(MPI)p 150 488 V 17 w(GET)p 239 488 V 17 w(CART\(COMM,)f(MAXDIMS,)h(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 544 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(PERIODS\(*\),)f(COORDS\(*\),)h(IERROR)75 635 y(MPI)p 150 635 V 17 w(CART)p 263 635 V 16 w(RANK\(COMM,)g(COORDS,) g(RANK,)g(IERROR\))170 692 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h (IERROR)75 783 y(MPI)p 150 783 V 17 w(CART)p 263 783 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g(COORDS,)f(IERROR\))170 839 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f(COORDS\(*\),)h(IERROR)75 930 y(MPI)p 150 930 V 17 w(GRAPH)p 287 930 V 16 w(NEIGHBORS)p 519 930 V 16 w(COUNT\(COMM,)g(RANK,)g(NNEIGHBORS,)f(IERROR\))170 987 y(INTEGER)h(COMM,)h(RANK,)f(NNEIGHBORS,)f(IERROR)75 1078 y(MPI)p 150 1078 V 17 w(GRAPH)p 287 1078 V 16 w(NEIGHBORS\(COMM,)g (RANK,)h(MAXNEIGHBORS,)f(NEIGHBORS,)h(IERROR\))170 1135 y(INTEGER)g(COMM,)h(RANK,)f(MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR)75 1226 y(MPI)p 150 1226 V 17 w(CART)p 263 1226 V 16 w(SHIFT\(COMM,)h (DIRECTION,)f(DISP,)h(RANK)p 1067 1226 V 17 w(SOURCE,)g(RANK)p 1371 1226 V 17 w(DEST,)g(IERROR\))170 1282 y(INTEGER)g(COMM,)h (DIRECTION,)e(DISP,)h(RANK)p 1009 1282 V 17 w(SOURCE,)g(RANK)p 1313 1282 V 16 w(DEST,)h(IERROR)75 1373 y(MPI)p 150 1373 V 17 w(CART)p 263 1373 V 16 w(SUB\(COMM,)f(REMAIN)p 662 1373 V 16 w(DIMS,)h(COMM)p 918 1373 V 16 w(NEW,)g(IERROR\))170 1430 y(INTEGER)f(COMM,)h(REMAIN)p 652 1430 V 16 w(DIMS\(*\),)f(COMM)p 979 1430 V 16 w(NEW,)h(IERROR)75 1521 y(MPI)p 150 1521 V 17 w(MAP)p 239 1521 V 17 w(CART\(COMM)p 472 1521 V 15 w(OLD,)g(NDIMS,)f(DIMS,)g(PERIOD,)g(NEWRANK,)g(IERROR\))170 1577 y(INTEGER)g(COMM)p 460 1577 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g (PERIOD\(*\),)g(NEWRANK,)f(IERROR)75 1668 y(MPI)p 150 1668 V 17 w(MAP)p 239 1668 V 17 w(GRAPH\(COMM)p 496 1668 V 15 w(OLD,)i(NNODES,)f(INDEX,)g(EDGES,)g(NEWRANK,)f(IERROR\))170 1725 y(INTEGER)h(COMM)p 460 1725 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g (EDGES\(*\),)f(NEWRANK,)h(IERROR)75 1925 y Ft(A.12)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Environmental)d(Inquiry)75 2035 y Fo(MPI)p 150 2035 V 17 w(GET)p 239 2035 V 17 w(VALID)p 376 2035 V 16 w(TAG)p 464 2035 V 17 w(RANGE\(LOW,)k(HIGH,)h(IERROR\)) 170 2092 y(INTEGER)g(LOW,)h(HIGH,)f(IERROR)75 2183 y(MPI)p 150 2183 V 17 w(GET)p 239 2183 V 17 w(PROCESSOR)p 472 2183 V 15 w(NAME\(NAME,)g(IERROR\))170 2239 y(CHARACTER*\(*\))g(NAME) 170 2296 y(INTEGER)g(IERROR)75 2387 y(MPI)p 150 2387 V 17 w(GET)p 239 2387 V 17 w(HOST\(COMM,)f(RANK,)h(IERROR\))170 2443 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 2534 y(MPI)p 150 2534 V 17 w(GET)p 239 2534 V 17 w(BUFFER)p 400 2534 V 16 w(PARAMS\(NUMSEND,)f(SIZESEND,)g(NUMRECV,)h(SIZERECV,)g (TOTALNUMSEDND,)393 2591 y(TOTALSIZESEN,)f(TOTALNUMRECV,)g (TOTALSIZERECV,)g(IERROR\))170 2647 y(INTEGER)h(NUMSEND,)g(SIZESEND,)g (NUMRECV,)g(SIZERECV,)f(TOTALNUMSEDND,)170 2704 y(TOTALSIZESEN,)h (TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)-32 46 y Fz(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 192 200 192 199 bop 75 -100 a FC(192)987 b Fs(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fo(MPI)p 150 45 15 2 v 17 w(SUGGEST)p 335 45 V 16 w(BUFFER)p 495 45 V 16 w(PARAMS\(NUMSEND,)22 b(SIZESEND,)h(NUMRECV,)g(SIZERECV,)393 102 y(TOTALNUMSEND,)f(TOTALSIZESEN,)g(TOTALNUMRECV,)g(TOTALSIZERECV,)g (FLAG,)393 158 y(IERROR\))170 214 y(INTEGER)h(NUMSEND,)g(SIZESEND,)g (NUMRECV,)g(SIZERECV,)f(TOTALNUMSEND,)170 271 y(TOTALSIZESEN,)h (TOTALNUMRECV,)f(TOTALSIZERECV,)g(IERROR)170 327 y(LOGICAL)h(FLAG)75 414 y(MPI)p 150 414 V 17 w(USER)p 263 414 V 16 w(SPECIFIES)p 495 414 V 16 w(BUFFER\(COMM,)g(BUFFER,)f(SIZE,)i(IERROR\))170 470 y()f(BUFFER\(*\))170 527 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR) 75 613 y(MPI)p 150 613 V 17 w(GET)p 239 613 V 17 w(ERRORMODE\(COMM,)e (MODE,)j(IERROR\))170 670 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)75 756 y(MPI)p 150 756 V 17 w(SET)p 239 756 V 17 w(ERRORMODE\(COMM,)e (MODE,)j(IERROR\))170 813 y(INTEGER)f(COMM,)h(MODE,)f(IERROR)75 899 y(MPI)p 150 899 V 17 w(SET)p 239 899 V 17 w(ERRHANDLER\(COMM,)e (HANDLER,)i(IERROR\))170 955 y(EXTERNAL)g(HANDLER)170 1012 y(INTEGER)g(COMM,)h(IERROR)75 1098 y(MPI)p 150 1098 V 17 w(ERROR)p 287 1098 V 16 w(STRING\(ERRORCODE,)e(STRING,)h(IERROR\)) 170 1155 y(INTEGER)g(ERRORCODE,)g(IERROR)170 1211 y(CHARACTER*\(*\))g (STRING)75 1298 y(MPI)p 150 1298 V 17 w(INIT\(IERROR\))170 1354 y(INTEGER)g(IERROR)75 1441 y(MPI)p 150 1441 V 17 w(FINALIZE\(IERROR\))170 1497 y(INTEGER)g(IERROR)75 1584 y(MPI)p 150 1584 V 17 w(INITIALIZED\(FLAG,)e(IERROR\))170 1640 y(LOGICAL)i(FLAG)170 1696 y(INTEGER)g(IERROR)75 1783 y(MPI)p 150 1783 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1839 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 193 201 193 200 bop 75 283 a Fy(MPI)40 b(F)-10 b(unction)40 b(Index)75 490 y FC(MPI)p 167 490 14 2 v 16 w(ABOR)l(T,)16 b(161)75 547 y(MPI)p 167 547 V 16 w(ADDRESS,)g(52)75 603 y(MPI)p 167 603 V 16 w(ALLGA)l(THER,)g(86)75 660 y(MPI)p 167 660 V 16 w(ALLGA)l(THER)-5 b(V,)16 b(87)75 716 y(MPI)p 167 716 V 16 w(ALLREDUCE,)g(96)75 773 y(MPI)p 167 773 V 16 w(ALL)l(TO)o(ALL,)h(88)75 829 y(MPI)p 167 829 V 16 w(ALL)l(TO)o(ALL)-5 b(V,)17 b(89)75 886 y(MPI)p 167 886 V 16 w(A)l(TTR)p 312 886 V 17 w(CREA)l(TE)p 520 886 V 17 w(KEY,)e(121)75 942 y(MPI)p 167 942 V 16 w(A)l(TTR)p 312 942 V 17 w(FREE)p 454 942 V 17 w(KEY,)g(122)75 1038 y(MPI)p 167 1038 V 16 w(BARRIER,)i(71)75 1095 y(MPI)p 167 1095 V 16 w(BCAST,)e(73)75 1191 y(MPI)p 167 1191 V 16 w(CANCEL,)g(37)75 1247 y(MPI)p 167 1247 V 16 w(CAR)l(T)p 312 1247 V 17 w(COORDS,)h(147)75 1303 y(MPI)p 167 1303 V 16 w(CAR)l(T)p 312 1303 V 17 w(RANK,)g(146)75 1360 y(MPI)p 167 1360 V 16 w(CAR)l(T)p 312 1360 V 17 w(SHIFT,)f(148)75 1416 y(MPI)p 167 1416 V 16 w(CAR)l(T)p 312 1416 V 17 w(SUB,)g(149)75 1473 y(MPI)p 167 1473 V 16 w(COMM)p 335 1473 V 16 w(DUP)l(,)g(113)75 1529 y(MPI)p 167 1529 V 16 w(COMM)p 335 1529 V 16 w(FREE,)g(115)75 1586 y(MPI)p 167 1586 V 16 w(COMM)p 335 1586 V 16 w(GR)o(OUP)l(,)g(113)75 1642 y(MPI)p 167 1642 V 16 w(COMM)p 335 1642 V 16 w(INIT,)h(67)75 1699 y(MPI)p 167 1699 V 16 w(COMM)p 335 1699 V 16 w(MAKE,)f(114)75 1755 y(MPI)p 167 1755 V 16 w(COMM)p 335 1755 V 16 w(RANK,)h(113)75 1812 y(MPI)p 167 1812 V 16 w(COMM)p 335 1812 V 16 w(SIZE,)f(113)75 1868 y(MPI)p 167 1868 V 16 w(COMM)p 335 1868 V 16 w(SPLIT,)h(114)75 1924 y(MPI)p 167 1924 V 16 w(COMM)p 335 1924 V 16 w(TEST)p 473 1924 V 16 w(INTER,)g(117)75 1981 y(MPI)p 167 1981 V 16 w(CREA)l(TE)p 374 1981 V 17 w(RECV,)f(40)75 2037 y(MPI)p 167 2037 V 16 w(CREA)l(TE)p 374 2037 V 17 w(RSEND,)g(39)75 2094 y(MPI)p 167 2094 V 16 w(CREA)l(TE)p 374 2094 V 17 w(SEND,)g(39)75 2150 y(MPI)p 167 2150 V 16 w(CREA)l(TE)p 374 2150 V 17 w(SSEND,)g(40)75 2246 y(MPI)p 167 2246 V 16 w(DELETE)p 372 2246 V 17 w(A)l(TTR,)g(123)75 2342 y(MPI)p 167 2342 V 16 w(ERR)o(OR)p 347 2342 V 18 w(STRING,)h(160)75 2438 y(MPI)p 167 2438 V 16 w(FINALIZE,)g(161)75 2534 y(MPI)p 167 2534 V 16 w(GA)l(THER,)f(73)75 2591 y(MPI)p 167 2591 V 16 w(GA)l(THER)-5 b(V,)15 b(74)75 2647 y(MPI)p 167 2647 V 16 w(GET)p 283 2647 V 16 w(A)l(TTR,)g(122)75 2704 y(MPI)p 167 2704 V 16 w(GET)p 283 2704 V 16 w(BUFFER)p 489 2704 V 17 w(P)l(ARAMS,)g(155)1048 490 y(MPI)p 1140 490 V 16 w(GET)p 1256 490 V 16 w(CAR)l(T,)g(146)1048 547 y(MPI)p 1140 547 V 16 w(GET)p 1256 547 V 16 w(CAR)l(TDIM,)g(145) 1048 605 y(MPI)p 1140 605 V 16 w(GET)p 1256 605 V 16 w(COUNT,)g(19)1048 662 y(MPI)p 1140 662 V 16 w(GET)p 1256 662 V 16 w(ELEMENTS,)g(57)1048 719 y(MPI)p 1140 719 V 16 w(GET)p 1256 719 V 16 w(ERR)o(ORMODE,)h(159)1048 776 y(MPI)p 1140 776 V 16 w(GET)p 1256 776 V 16 w(GRAPH,)f(145)1048 833 y(MPI)p 1140 833 V 16 w(GET)p 1256 833 V 16 w(GRAPHDIMS,)g(145)1048 890 y(MPI)p 1140 890 V 16 w(GET)p 1256 890 V 16 w(HOST,)g(154)1048 947 y(MPI)p 1140 947 V 16 w(GET)p 1256 947 V 16 w(PR)o(OCESSOR)p 1552 947 V 18 w(NAME,)f(153)1048 1004 y(MPI)p 1140 1004 V 16 w(GET)p 1256 1004 V 16 w(SOUR)o(CE,)h(18)1048 1061 y(MPI)p 1140 1061 V 16 w(GET)p 1256 1061 V 16 w(T)l(A)o(G,)f(18)1048 1118 y(MPI)p 1140 1118 V 16 w(GET)p 1256 1118 V 16 w(V)-5 b(ALID)p 1414 1118 V 17 w(T)l(A)o(G)p 1529 1118 V 16 w(RANGE,)15 b(153)1048 1175 y(MPI)p 1140 1175 V 16 w(GRAPH)p 1324 1175 V 17 w(NEIGHBORS,)h(147)1048 1232 y(MPI)p 1140 1232 V 16 w(GRAPH)p 1324 1232 V 17 w(NEIGHBORS)p 1617 1232 V 17 w(COUNT,)10 b(147)1048 1289 y(MPI)p 1140 1289 V 16 w(GR)o(OUP)p 1324 1289 V 17 w(DIFFERENCE,)k(109)1048 1346 y(MPI)p 1140 1346 V 16 w(GR)o(OUP)p 1324 1346 V 17 w(EX)o(CL,)g(110)1048 1403 y(MPI)p 1140 1403 V 16 w(GR)o(OUP)p 1324 1403 V 17 w(FREE,)g(112)1048 1460 y(MPI)p 1140 1460 V 16 w(GR)o(OUP)p 1324 1460 V 17 w(INCL,)h(110)1048 1517 y(MPI)p 1140 1517 V 16 w(GR)o(OUP)p 1324 1517 V 17 w(INTERSECTION,)h(109)1048 1574 y(MPI)p 1140 1574 V 16 w(GR)o(OUP)p 1324 1574 V 17 w(RANGE)p 1509 1574 V 16 w(EX)o(CL,)f(111)1048 1631 y(MPI)p 1140 1631 V 16 w(GR)o(OUP)p 1324 1631 V 17 w(RANGE)p 1509 1631 V 16 w(INCL,)h(111)1048 1688 y(MPI)p 1140 1688 V 16 w(GR)o(OUP)p 1324 1688 V 17 w(RANK,)f(108)1048 1745 y(MPI)p 1140 1745 V 16 w(GR)o(OUP)p 1324 1745 V 17 w(SIZE,)g(107)1048 1802 y(MPI)p 1140 1802 V 16 w(GR)o(OUP)p 1324 1802 V 17 w(TRANSLA)l(TE)p 1622 1802 V 17 w(RANKS,)c(108)1048 1859 y(MPI)p 1140 1859 V 16 w(GR)o(OUP)p 1324 1859 V 17 w(UNION,)k(108)1048 1968 y(MPI)p 1140 1968 V 16 w(INIT,)h(160)1048 2025 y(MPI)p 1140 2025 V 16 w(INITIALIZED,)g(161)1048 2082 y(MPI)p 1140 2082 V 16 w(INTER)o(COMM)p 1454 2082 V 16 w(FINISH,)g(119)1048 2139 y(MPI)p 1140 2139 V 16 w(INTER)o(COMM)p 1454 2139 V 16 w(MER)o(GE,)f(119)1048 2196 y(MPI)p 1140 2196 V 16 w(INTER)o(COMM)p 1454 2196 V 16 w(NAME)p 1611 2196 V 17 w(ST)l(AR)l(T,)g(120)1048 2253 y(MPI)p 1140 2253 V 16 w(INTER)o(COMM)p 1454 2253 V 16 w(ST)l(AR)l(T,)h(118)1048 2310 y(MPI)p 1140 2310 V 16 w(IPR)o(OBE,)g(36)1048 2367 y(MPI)p 1140 2367 V 16 w(IRECV,)g(29)1048 2424 y(MPI)p 1140 2424 V 16 w(IRSEND,)g(28)1048 2481 y(MPI)p 1140 2481 V 16 w(ISEND,)f(28)1048 2538 y(MPI)p 1140 2538 V 16 w(ISSEND,)g(29)1048 2647 y(MPI)p 1140 2647 V 16 w(MAKE)p 1298 2647 V 16 w(CAR)l(T,)h(141)1048 2704 y(MPI)p 1140 2704 V 16 w(MAKE)p 1298 2704 V 16 w(DIMS,)f(142)941 2828 y(193)p eop %%Page: 194 202 194 201 bop 75 -100 a FC(194)1326 b Fs(MPI)15 b(F)l(unction)h(Index)75 45 y FC(MPI)p 167 45 14 2 v 16 w(MAKE)p 325 45 V 17 w(GRAPH,)f(143)75 102 y(MPI)p 167 102 V 16 w(MAP)p 290 102 V 16 w(CAR)l(T,)h(150)75 158 y(MPI)p 167 158 V 16 w(MAP)p 290 158 V 16 w(GRAPH,)f(151)75 260 y(MPI)p 167 260 V 16 w(PR)o(OBE,)h(37)75 316 y(MPI)p 167 316 V 16 w(PUT)p 281 316 V 17 w(A)l(TTR,)f(122)75 417 y(MPI)p 167 417 V 16 w(RECV,)g(17)75 474 y(MPI)p 167 474 V 16 w(REDUCE,)g(90)75 531 y(MPI)p 167 531 V 16 w(REDUCE)p 380 531 V 17 w(SCA)l(TTER,)g(97)75 587 y(MPI)p 167 587 V 16 w(REQUEST)p 405 587 V 18 w(FREE,)f(41)75 644 y(MPI)p 167 644 V 16 w(RSEND,)i(25)75 745 y(MPI)p 167 745 V 16 w(SCAN,)f(99)75 802 y(MPI)p 167 802 V 16 w(SCA)l(TTER,)h(82)75 859 y(MPI)p 167 859 V 16 w(SCA)l(TTER)-5 b(V,)16 b(83)75 915 y(MPI)p 167 915 V 16 w(SEND,)f(15)75 972 y(MPI)p 167 972 V 16 w(SENDRECV,)g(42)75 1028 y(MPI)p 167 1028 V 16 w(SENDRECV)p 439 1028 V 17 w(REPLA)o(CE,)g(43)75 1085 y(MPI)p 167 1085 V 16 w(SET)p 272 1085 V 17 w(ERRHANDLER,)h(159)75 1142 y(MPI)p 167 1142 V 16 w(SET)p 272 1142 V 17 w(ERR)o(ORMODE,)g(159) 75 1198 y(MPI)p 167 1198 V 16 w(SSEND,)f(26)75 1255 y(MPI)p 167 1255 V 16 w(ST)l(AR)l(T,)h(40)75 1312 y(MPI)p 167 1312 V 16 w(ST)l(AR)l(T)l(ALL,)h(41)75 1368 y(MPI)p 167 1368 V 16 w(SUGGEST)p 403 1368 V 16 w(BUFFER)p 609 1368 V 17 w(P)l(ARAMS,)10 b(156)75 1470 y(MPI)p 167 1470 V 16 w(TEST,)15 b(30)75 1526 y(MPI)p 167 1526 V 16 w(TEST)p 305 1526 V 17 w(CANCELLED,)g(38)75 1583 y(MPI)p 167 1583 V 16 w(TEST)l(ALL,)h(34)75 1639 y(MPI)p 167 1639 V 16 w(TEST)l(ANY,)f(33)75 1696 y(MPI)p 167 1696 V 16 w(TOPO)p 317 1696 V 17 w(ST)l(A)l(TUS,)h(144)75 1753 y(MPI)p 167 1753 V 16 w(TYPE)p 312 1753 V 17 w(COMMIT,)e(54)75 1809 y(MPI)p 167 1809 V 16 w(TYPE)p 312 1809 V 17 w(CONTIGUOUS,)h(46)75 1866 y(MPI)p 167 1866 V 16 w(TYPE)p 312 1866 V 17 w(COUNT,)g(53)75 1923 y(MPI)p 167 1923 V 16 w(TYPE)p 312 1923 V 17 w(EXTENT,)f(52)75 1979 y(MPI)p 167 1979 V 16 w(TYPE)p 312 1979 V 17 w(FREE,)g(54)75 2036 y(MPI)p 167 2036 V 16 w(TYPE)p 312 2036 V 17 w(HINDEXED,)g(50)75 2093 y(MPI)p 167 2093 V 16 w(TYPE)p 312 2093 V 17 w(HVECTOR,)h(48)75 2149 y(MPI)p 167 2149 V 16 w(TYPE)p 312 2149 V 17 w(INDEXED,)f(49)75 2206 y(MPI)p 167 2206 V 16 w(TYPE)p 312 2206 V 17 w(SIZE,)h(53)75 2263 y(MPI)p 167 2263 V 16 w(TYPE)p 312 2263 V 17 w(STR)o(UCT,)f(51)75 2319 y(MPI)p 167 2319 V 16 w(TYPE)p 312 2319 V 17 w(VECTOR,)h(47)75 2420 y(MPI)p 167 2420 V 16 w(USER)p 306 2420 V 17 w(ALLREDUCE,)h(96)75 2477 y(MPI)p 167 2477 V 16 w(USER)p 306 2477 V 17 w(ALLREDUCEA,)g(97)75 2534 y(MPI)p 167 2534 V 16 w(USER)p 306 2534 V 17 w(REDUCE,)f(93)75 2590 y(MPI)p 167 2590 V 16 w(USER)p 306 2590 V 17 w(REDUCE)p 520 2590 V 17 w(SCA)l(TTER,)g(98)75 2647 y(MPI)p 167 2647 V 16 w(USER)p 306 2647 V 17 w(REDUCE)p 520 2647 V 17 w(SCA)l(TTERA,)g(98)75 2704 y(MPI)p 167 2704 V 16 w(USER)p 306 2704 V 17 w(REDUCEA,)g(95)1048 45 y(MPI)p 1140 45 V 16 w(USER)p 1279 45 V 17 w(SCAN,)g(99)1048 102 y(MPI)p 1140 102 V 16 w(USER)p 1279 102 V 17 w(SCANA,)g(100)1048 158 y(MPI)p 1140 158 V 16 w(USER)p 1279 158 V 17 w(SPECIFIES)p 1534 158 V 17 w(BUFFER,)g(157)1048 256 y(MPI)p 1140 256 V 16 w(W)-5 b(AIT,)15 b(30)1048 312 y(MPI)p 1140 312 V 16 w(W)-5 b(AIT)l(ALL,)16 b(34)1048 369 y(MPI)p 1140 369 V 16 w(W)-5 b(AIT)l(ANY,)15 b(33)1967 46 y Fz(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .